Examples with JobId org.ow2.proactive.scheduler.common.job.JobId used on opensource projects

Example 16 with JobId

use of org.ow2.proactive.scheduler.common.job.JobId in project scheduling by ow2-proactive.

the class TestGlobalSpace method testGlobalSpace.

@Test
public void testGlobalSpace() throws Throwable {
    File in = tmpFolder.newFolder("input_space");
    String inPath = in.getAbsolutePath();
    File out = tmpFolder.newFolder("output_space");
    String outPath = out.getAbsolutePath();
    writeFiles(inFiles, inPath);
    TaskFlowJob job = new TaskFlowJob();
    job.setName(this.getClass().getSimpleName());
    job.setInputSpace(in.toURI().toURL().toString());
    job.setOutputSpace(out.toURI().toURL().toString());
    JavaTask A = new JavaTask();
    A.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    A.setName("A");
    for (String[] file : inFiles) {
        A.addInputFiles(file[0], InputAccessMode.TransferFromInputSpace);
        A.addOutputFiles(file[0] + ".glob.A", OutputAccessMode.TransferToGlobalSpace);
    }
    A.setPreScript(new SimpleScript(scriptA, "groovy"));
    A.setForkEnvironment(new ForkEnvironment());
    job.addTask(A);
    JavaTask B = new JavaTask();
    B.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    B.setName("B");
    B.addDependence(A);
    for (String[] file : inFiles) {
        B.addInputFiles(file[0] + ".glob.A", InputAccessMode.TransferFromGlobalSpace);
        B.addOutputFiles(file[0] + ".out", OutputAccessMode.TransferToOutputSpace);
    }
    B.setPreScript(new SimpleScript(scriptB, "groovy"));
    B.setForkEnvironment(new ForkEnvironment());
    job.addTask(B);
    Scheduler scheduler = schedulerHelper.getSchedulerInterface();
    JobId id = scheduler.submit(job);
    schedulerHelper.waitForEventJobFinished(id);
    assertFalse(schedulerHelper.getJobResult(id).hadException());
    /**
     * check: inFiles > IN > LOCAL A > GLOBAL > LOCAL B > OUT
     */
    for (String[] inFile : inFiles) {
        File f = new File(outPath + File.separator + inFile[0] + ".out");
        assertTrue("File does not exist: " + f.getAbsolutePath(), f.exists());
        Assert.assertEquals("Original and copied files differ", inFile[1], FileUtils.readFileToString(f));
        f.delete();
        File inf = new File(inPath + File.separator + inFile[0]);
        inf.delete();
    }
    /**
     * check that the file produced is accessible in the global user space via the scheduler API
     */
    String globalURI = scheduler.getGlobalSpaceURIs().get(0);
    assertTrue(globalURI.startsWith("file:"));
    String globalPath = new File(new URI(globalURI)).getAbsolutePath();
    FileSystemManager fsManager = VFSFactory.createDefaultFileSystemManager();
    for (String[] file : inFiles) {
        FileObject outFile = fsManager.resolveFile(globalURI + "/" + file[0] + ".glob.A");
        log("Checking existence of " + outFile.getURL());
        assertTrue(outFile.getURL() + " exists", outFile.exists());
        File outFile2 = new File(globalPath, file[0] + ".glob.A");
        log("Checking existence of " + outFile2);
        assertTrue(outFile2 + " exists", outFile2.exists());
    }
}

Example 17 with JobId

use of org.ow2.proactive.scheduler.common.job.JobId in project scheduling by ow2-proactive.

the class TestJobDataspaceSubmission method testJobDataspaceSubmission.

/**
 * Tests start here.
 *
 * @throws Throwable any exception that can be thrown during the test.
 */
@org.junit.Test
public void testJobDataspaceSubmission() throws Throwable {
    // create initial directories and files
    setup();
    TaskFlowJob job = new TaskFlowJob();
    FileSystemServerDeployer filesServerIn = new FileSystemServerDeployer(IN, IOSPACE + IN, true, true);
    String url = filesServerIn.getVFSRootURL();
    job.setInputSpace(url);
    FileSystemServerDeployer filesServerOut = new FileSystemServerDeployer(OUT, IOSPACE + OUT, true);
    url = filesServerOut.getVFSRootURL();
    job.setOutputSpace(url);
    FileSystemServerDeployer filesServerGlob = new FileSystemServerDeployer(GLOB, IOSPACE + GLOB, true);
    url = filesServerGlob.getVFSRootURL();
    job.setGlobalSpace(url);
    FileSystemServerDeployer filesServerUser = new FileSystemServerDeployer(USER, IOSPACE + USER, true);
    url = filesServerUser.getVFSRootURL();
    job.setUserSpace(url);
    job.setName(this.getClass().getSimpleName());
    NativeTask t1 = new NativeTask();
    t1.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t1.addOutputFiles(out1, OutputAccessMode.TransferToOutputSpace);
    t1.setName("native_java1");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t1.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " > " + out1 });
            break;
        case unix:
            t1.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " > " + out1 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t1);
    NativeTask t2 = new NativeTask();
    t2.addInputFiles(in2, InputAccessMode.TransferFromOutputSpace);
    t2.addOutputFiles(out2, OutputAccessMode.TransferToOutputSpace);
    t2.setName("native_java2");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t2.setCommandLine(new String[] { "cmd", "/C", "type " + in2 + " > " + out2 });
            break;
        case unix:
            t2.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in2 + " > " + out2 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t2);
    NativeTask t3 = new NativeTask();
    t3.addInputFiles(in3, InputAccessMode.TransferFromGlobalSpace);
    t3.addOutputFiles(out3, OutputAccessMode.TransferToGlobalSpace);
    t3.setName("native_java3");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t3.setCommandLine(new String[] { "cmd", "/C", "type " + in3 + " > " + out3 });
            break;
        case unix:
            t3.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in3 + " > " + out3 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t3);
    NativeTask t4 = new NativeTask();
    t4.addInputFiles(in4, InputAccessMode.TransferFromUserSpace);
    t4.addOutputFiles(out4, OutputAccessMode.TransferToUserSpace);
    t4.setName("native_java4");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t4.setCommandLine(new String[] { "cmd", "/C", "type " + in4 + " > " + out4 });
            break;
        case unix:
            t4.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in4 + " > " + out4 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t4);
    NativeTask t10 = new NativeTask();
    t10.addDependence(t2);
    t10.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t10.addInputFiles(new FileSelector("*b.txt"), InputAccessMode.TransferFromOutputSpace);
    t10.addOutputFiles("*aa.txt", OutputAccessMode.TransferToOutputSpace);
    t10.setName("native_java10");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t10.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out2 + " > " + out10 });
            break;
        case unix:
            t10.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out2 + " > " + out10 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t10);
    NativeTask t11 = new NativeTask();
    t11.addDependence(t3);
    t11.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t11.addInputFiles(new FileSelector("*c.txt"), InputAccessMode.TransferFromGlobalSpace);
    t11.addOutputFiles("*bb.txt", OutputAccessMode.TransferToGlobalSpace);
    t11.setName("native_java11");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t11.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out3 + " > " + out11 });
            break;
        case unix:
            t11.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out3 + " > " + out11 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t11);
    NativeTask t12 = new NativeTask();
    t12.addDependence(t4);
    t12.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t12.addInputFiles(new FileSelector("*d.txt"), InputAccessMode.TransferFromUserSpace);
    t12.addOutputFiles("*cc.txt", OutputAccessMode.TransferToUserSpace);
    t12.setName("native_java7");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t12.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out4 + " > " + out12 });
            break;
        case unix:
            t12.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out4 + " > " + out12 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t12);
    JobId id = schedulerHelper.testJobSubmission(job);
    // check results are 0
    JobResult res = schedulerHelper.getJobResult(id);
    Assert.assertFalse(schedulerHelper.getJobResult(id).hadException());
    for (Map.Entry<String, TaskResult> entry : res.getAllResults().entrySet()) {
        Assert.assertEquals(0, entry.getValue().value());
    }
    // remove job
    schedulerHelper.removeJob(id);
    schedulerHelper.waitForEventJobRemoved(id);
    // check files
    File fout = new File(outputDir.getAbsolutePath() + File.separator + out1);
    Assert.assertEquals(in1, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out2);
    Assert.assertEquals(in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out3);
    Assert.assertEquals(in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out4);
    Assert.assertEquals(in4, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out10);
    Assert.assertEquals(in1 + in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out11);
    Assert.assertEquals(in1 + in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out12);
    Assert.assertEquals(in1 + in4, getContent(fout));
    // fout = new File(outputDir.getAbsolutePath() + File.separator + out20);
    // Assert.assertEquals(in1 + in2, getContent(fout));
    // fout = new File(globalDir.getAbsolutePath() + File.separator + out21);
    // Assert.assertEquals(in1 + in3, getContent(fout));
    // fout = new File(userDir.getAbsolutePath() + File.separator + out22);
    // Assert.assertEquals(in1 + in4, getContent(fout));
    filesServerIn.terminate();
    filesServerOut.terminate();
    filesServerGlob.terminate();
    filesServerUser.terminate();
    // and clean tmp space
    clean();
}

Example 18 with JobId

use of org.ow2.proactive.scheduler.common.job.JobId in project scheduling by ow2-proactive.

the class TestWorkflowDataspace method testJavaTask.

private void testJavaTask() throws Throwable {
    String tmpdir = System.getProperty("java.io.tmpdir");
    File inputSpace = new File(tmpdir + File.separator + "inputSpace." + System.currentTimeMillis());
    File outputSpace = new File(tmpdir + File.separator + "outputSpace." + System.currentTimeMillis());
    inputSpace.mkdir();
    outputSpace.mkdir();
    for (int it = 0; it < 3; it++) {
        for (int dup = 0; dup < 3; dup++) {
            File f = new File(inputSpace.getCanonicalPath() + File.separator + it + "_" + dup + ".in");
            f.createNewFile();
            BufferedWriter out = new BufferedWriter(new FileWriter(f));
            out.write("it " + it + " dup " + dup + "\n");
            out.close();
        }
    }
    TaskFlowJob job = new TaskFlowJob();
    job.setName(TestWorkflowDataspace.class.getSimpleName());
    job.setInputSpace(inputSpace.toURI().toURL().toString());
    job.setOutputSpace(outputSpace.toURI().toURL().toString());
    ForkEnvironment forkEnvironment = new ForkEnvironment();
    JavaTask t = new JavaTask();
    t.setName("T");
    t.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    t.setFlowScript(FlowScript.createReplicateFlowScript("runs = 3;"));
    t.setFlowBlock(FlowBlock.START);
    t.setForkEnvironment(forkEnvironment);
    job.addTask(t);
    JavaTask t1 = new JavaTask();
    t1.setName("T1");
    t1.setForkEnvironment(forkEnvironment);
    t1.setExecutableClassName(JobWorkflowDataspace.class.getCanonicalName());
    t1.setForkEnvironment(new ForkEnvironment());
    t1.addDependence(t);
    t1.addInputFiles("$PA_TASK_ITERATION_$PA_TASK_REPLICATION.in", InputAccessMode.TransferFromInputSpace);
    t1.addOutputFiles("$PA_TASK_ITERATION_$PA_TASK_REPLICATION.out", OutputAccessMode.TransferToOutputSpace);
    job.addTask(t1);
    JavaTask t2 = new JavaTask();
    t2.setName("T2");
    t2.setForkEnvironment(forkEnvironment);
    t2.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    t2.addDependence(t1);
    t2.setFlowScript(// 
    FlowScript.createLoopFlowScript(// 
    "if (variables.get('PA_TASK_ITERATION') < 2) {" + // 
    "loop = true;" + // 
    "} else {" + // 
    "loop = false;" + "}", "T"));
    t2.setFlowBlock(FlowBlock.END);
    job.addTask(t2);
    JobId id = TWorkflowJobs.testJobSubmission(schedulerHelper, job, null);
    Assert.assertFalse(schedulerHelper.getJobResult(id).hadException());
    for (int it = 0; it < 3; it++) {
        for (int dup = 0; dup < 3; dup++) {
            File f = new File(outputSpace.getCanonicalPath() + File.separator + it + "_" + dup + ".out");
            Assert.assertTrue("Missing output file " + f.getName(), f.exists());
            BufferedReader in = new BufferedReader(new FileReader(f));
            String line = in.readLine();
            Assert.assertTrue("Wrong content for " + f.getCanonicalPath(), line.equals("it " + it + " dup " + dup));
        }
    }
}

Example 19 with JobId

use of org.ow2.proactive.scheduler.common.job.JobId in project scheduling by ow2-proactive.

the class SmartProxyImpl method downloadTaskOutputFiles.

@Override
protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, String t_name, String localFolder) throws Exception {
    AwaitedTask atask = awaitedjob.getAwaitedTask(t_name);
    if (atask == null) {
        throw new IllegalArgumentException("The task " + t_name + " does not belong to job " + jobId + " or has already been removed");
    }
    if (atask.isTransferring()) {
        log.warn("The task " + t_name + " of job " + jobId + " is already transferring its output");
        return;
    }
    String pull_URL = awaitedjob.getPullURL();
    if (awaitedjob.isIsolateTaskOutputs()) {
        pull_URL = pull_URL.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + atask.getTaskId());
    }
    FileObject remotePullFolderFO;
    FileObject localfolderFO;
    try {
        remotePullFolderFO = jobTracker.resolveFile(pull_URL);
        localfolderFO = jobTracker.resolveFile(localFolder);
    } catch (FileSystemException e) {
        log.error("Could not retrieve data for job " + jobId, e);
        throw new IllegalStateException("Could not retrieve data for job " + jobId, e);
    }
    String sourceUrl = remotePullFolderFO.getURL().toString();
    String destUrl = localfolderFO.getURL().toString();
    org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fileSelector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
    List<OutputSelector> ouputFileSelectors = atask.getOutputSelectors();
    for (OutputSelector os : ouputFileSelectors) {
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fs = os.getOutputFiles();
        if (!fs.getIncludes().isEmpty()) {
            fileSelector.addIncludes(fs.getIncludes());
        }
        if (!fs.getExcludes().isEmpty()) {
            fileSelector.addExcludes(fs.getExcludes());
        }
    }
    if (log.isDebugEnabled()) {
        log.debug("Looking at files in " + sourceUrl + " with " + fileSelector.getIncludes() + "-" + fileSelector.getExcludes());
        boolean goon = true;
        int cpt = 0;
        FileObject[] fos = null;
        while (goon) {
            fos = remotePullFolderFO.findFiles(fileSelector);
            goon = cpt < 50 && (fos == null || fos.length == 0);
            cpt++;
            if (goon) {
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        }
        if (fos != null && fos.length > 0) {
            for (FileObject fo : fos) {
                log.debug("Found " + fo.getName());
            }
        } else {
            log.warn("Couldn't find " + fileSelector.getIncludes() + "-" + fileSelector.getExcludes() + " in " + sourceUrl);
        }
    }
    if (awaitedjob.isAutomaticTransfer()) {
        DataTransferProcessor dtp = new DataTransferProcessor(remotePullFolderFO, localfolderFO, jobId, t_name, fileSelector);
        jobTracker.setTaskTransferring(jobId, t_name, true);
        threadPool.submit((Runnable) dtp);
    } else {
        log.debug("Copying files from " + sourceUrl + " to " + destUrl);
        try {
            localfolderFO.copyFrom(remotePullFolderFO, fileSelector);
        } catch (FileSystemException e) {
            log.error(e);
            throw e;
        } finally {
            jobTracker.setTaskTransferring(jobId, t_name, false);
        }
        // task is removed from the job tracker only if the transfer is successful
        jobTracker.removeAwaitedTask(jobId, t_name);
        log.debug("Finished copying files from " + sourceUrl + " to " + destUrl);
    // ok we can remove the task
    }
}

Example 20 with JobId

use of org.ow2.proactive.scheduler.common.job.JobId in project scheduling by ow2-proactive.

the class TestKillPendingTask method testKillPendingTasks.

@Test
public void testKillPendingTasks() throws Throwable {
    String task1Name = "task1";
    String task2Name = "task2";
    JobId id = schedulerHelper.submitJob(new File(jobDescriptor.toURI()).getAbsolutePath());
    // check events reception
    log("Job submitted, id " + id.toString());
    log("Waiting for jobSubmitted");
    schedulerHelper.waitForEventJobSubmitted(id);
    schedulerHelper.killTask(id.toString(), task1Name);
    log("Waiting for task finished : " + task1Name);
    schedulerHelper.waitForEventTaskFinished(id, task1Name);
    schedulerHelper.killTask(id.toString(), task2Name);
    log("Waiting for task finished : " + task2Name);
    schedulerHelper.waitForEventTaskFinished(id, task2Name);
    log("Waiting for job finished");
    schedulerHelper.waitForEventJobFinished(id);
    JobResult res = schedulerHelper.getJobResult(id);
    Map<String, TaskResult> results = res.getAllResults();
    // check that all tasks results are defined
    assertTrue(results.get(task1Name).hadException());
    assertNotNull(results.get(task1Name).getException());
    assertTrue(results.get(task2Name).hadException());
    assertNotNull(results.get(task2Name).getException());
}