Examples with FileSelector org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector used on opensource projects

Example 6 with FileSelector

use of org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector in project scheduling by ow2-proactive.

the class TestJobDataspaceSubmission method testJobDataspaceSubmission.

/**
 * Tests start here.
 *
 * @throws Throwable any exception that can be thrown during the test.
 */
@org.junit.Test
public void testJobDataspaceSubmission() throws Throwable {
    // create initial directories and files
    setup();
    TaskFlowJob job = new TaskFlowJob();
    FileSystemServerDeployer filesServerIn = new FileSystemServerDeployer(IN, IOSPACE + IN, true, true);
    String url = filesServerIn.getVFSRootURL();
    job.setInputSpace(url);
    FileSystemServerDeployer filesServerOut = new FileSystemServerDeployer(OUT, IOSPACE + OUT, true);
    url = filesServerOut.getVFSRootURL();
    job.setOutputSpace(url);
    FileSystemServerDeployer filesServerGlob = new FileSystemServerDeployer(GLOB, IOSPACE + GLOB, true);
    url = filesServerGlob.getVFSRootURL();
    job.setGlobalSpace(url);
    FileSystemServerDeployer filesServerUser = new FileSystemServerDeployer(USER, IOSPACE + USER, true);
    url = filesServerUser.getVFSRootURL();
    job.setUserSpace(url);
    job.setName(this.getClass().getSimpleName());
    NativeTask t1 = new NativeTask();
    t1.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t1.addOutputFiles(out1, OutputAccessMode.TransferToOutputSpace);
    t1.setName("native_java1");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t1.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " > " + out1 });
            break;
        case unix:
            t1.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " > " + out1 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t1);
    NativeTask t2 = new NativeTask();
    t2.addInputFiles(in2, InputAccessMode.TransferFromOutputSpace);
    t2.addOutputFiles(out2, OutputAccessMode.TransferToOutputSpace);
    t2.setName("native_java2");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t2.setCommandLine(new String[] { "cmd", "/C", "type " + in2 + " > " + out2 });
            break;
        case unix:
            t2.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in2 + " > " + out2 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t2);
    NativeTask t3 = new NativeTask();
    t3.addInputFiles(in3, InputAccessMode.TransferFromGlobalSpace);
    t3.addOutputFiles(out3, OutputAccessMode.TransferToGlobalSpace);
    t3.setName("native_java3");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t3.setCommandLine(new String[] { "cmd", "/C", "type " + in3 + " > " + out3 });
            break;
        case unix:
            t3.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in3 + " > " + out3 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t3);
    NativeTask t4 = new NativeTask();
    t4.addInputFiles(in4, InputAccessMode.TransferFromUserSpace);
    t4.addOutputFiles(out4, OutputAccessMode.TransferToUserSpace);
    t4.setName("native_java4");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t4.setCommandLine(new String[] { "cmd", "/C", "type " + in4 + " > " + out4 });
            break;
        case unix:
            t4.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in4 + " > " + out4 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t4);
    NativeTask t10 = new NativeTask();
    t10.addDependence(t2);
    t10.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t10.addInputFiles(new FileSelector("*b.txt"), InputAccessMode.TransferFromOutputSpace);
    t10.addOutputFiles("*aa.txt", OutputAccessMode.TransferToOutputSpace);
    t10.setName("native_java10");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t10.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out2 + " > " + out10 });
            break;
        case unix:
            t10.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out2 + " > " + out10 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t10);
    NativeTask t11 = new NativeTask();
    t11.addDependence(t3);
    t11.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t11.addInputFiles(new FileSelector("*c.txt"), InputAccessMode.TransferFromGlobalSpace);
    t11.addOutputFiles("*bb.txt", OutputAccessMode.TransferToGlobalSpace);
    t11.setName("native_java11");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t11.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out3 + " > " + out11 });
            break;
        case unix:
            t11.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out3 + " > " + out11 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t11);
    NativeTask t12 = new NativeTask();
    t12.addDependence(t4);
    t12.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t12.addInputFiles(new FileSelector("*d.txt"), InputAccessMode.TransferFromUserSpace);
    t12.addOutputFiles("*cc.txt", OutputAccessMode.TransferToUserSpace);
    t12.setName("native_java7");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t12.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out4 + " > " + out12 });
            break;
        case unix:
            t12.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out4 + " > " + out12 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t12);
    JobId id = schedulerHelper.testJobSubmission(job);
    // check results are 0
    JobResult res = schedulerHelper.getJobResult(id);
    Assert.assertFalse(schedulerHelper.getJobResult(id).hadException());
    for (Map.Entry<String, TaskResult> entry : res.getAllResults().entrySet()) {
        Assert.assertEquals(0, entry.getValue().value());
    }
    // remove job
    schedulerHelper.removeJob(id);
    schedulerHelper.waitForEventJobRemoved(id);
    // check files
    File fout = new File(outputDir.getAbsolutePath() + File.separator + out1);
    Assert.assertEquals(in1, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out2);
    Assert.assertEquals(in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out3);
    Assert.assertEquals(in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out4);
    Assert.assertEquals(in4, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out10);
    Assert.assertEquals(in1 + in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out11);
    Assert.assertEquals(in1 + in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out12);
    Assert.assertEquals(in1 + in4, getContent(fout));
    // fout = new File(outputDir.getAbsolutePath() + File.separator + out20);
    // Assert.assertEquals(in1 + in2, getContent(fout));
    // fout = new File(globalDir.getAbsolutePath() + File.separator + out21);
    // Assert.assertEquals(in1 + in3, getContent(fout));
    // fout = new File(userDir.getAbsolutePath() + File.separator + out22);
    // Assert.assertEquals(in1 + in4, getContent(fout));
    filesServerIn.terminate();
    filesServerOut.terminate();
    filesServerGlob.terminate();
    filesServerUser.terminate();
    // and clean tmp space
    clean();
}

Example 7 with FileSelector

use of org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector in project scheduling by ow2-proactive.

the class TestDataspaceSelectorsData method testSelectors.

@Test
public void testSelectors() throws Exception {
    TaskFlowJob jobDef = new TaskFlowJob();
    JavaTask task = createDefaultTask("task1");
    Set<String> inInclude1 = Sets.newHashSet("inInclude1_1", "inInclude1_2");
    Set<String> inExclude1 = Sets.newHashSet("inExclude1_1", "inExclude1_2");
    Set<String> outInclude1 = Sets.newHashSet("outInclude1_1", "outInclude1_2");
    Set<String> outExclude1 = Sets.newHashSet("outExclude1_1", "outExclude1_2");
    Set<String> inInclude2 = Sets.newHashSet("inInclude" + createString(500));
    Set<String> inExclude2 = Sets.newHashSet("inExclude" + createString(500));
    Set<String> outInclude2 = Sets.newHashSet("outInclude" + createString(500));
    Set<String> outExclude2 = Sets.newHashSet("outExclude" + createString(500));
    FileSelector fileSelector;
    task.addInputFiles(new FileSelector(inInclude1, inExclude1), InputAccessMode.TransferFromGlobalSpace);
    task.addInputFiles(new FileSelector(inInclude1, inExclude1), InputAccessMode.TransferFromUserSpace);
    fileSelector = new FileSelector(inInclude2, inExclude2);
    task.addInputFiles(fileSelector, InputAccessMode.TransferFromInputSpace);
    task.addOutputFiles(new FileSelector(outInclude1, outExclude1), OutputAccessMode.TransferToGlobalSpace);
    task.addOutputFiles(new FileSelector(outInclude1, outExclude1), OutputAccessMode.TransferToUserSpace);
    fileSelector = new FileSelector(outInclude2, outExclude2);
    task.addOutputFiles(fileSelector, OutputAccessMode.TransferToOutputSpace);
    jobDef.addTask(task);
    InternalJob job = defaultSubmitJobAndLoadInternal(true, jobDef);
    InternalTask task1 = job.getTask("task1");
    Assert.assertEquals(3, task1.getInputFilesList().size());
    Assert.assertEquals(InputAccessMode.TransferFromGlobalSpace, task1.getInputFilesList().get(0).getMode());
    Assert.assertEquals(InputAccessMode.TransferFromUserSpace, task1.getInputFilesList().get(1).getMode());
    Assert.assertEquals(3, task1.getOutputFilesList().size());
    checkSelector(task1.getInputFilesList().get(0).getInputFiles(), inInclude1, inExclude1, true);
    checkSelector(task1.getInputFilesList().get(1).getInputFiles(), inInclude1, inExclude1, true);
    checkSelector(task1.getInputFilesList().get(2).getInputFiles(), inInclude2, inExclude2, false);
    checkSelector(task1.getOutputFilesList().get(0).getOutputFiles(), outInclude1, outExclude1, true);
    checkSelector(task1.getOutputFilesList().get(1).getOutputFiles(), outInclude1, outExclude1, true);
    checkSelector(task1.getOutputFilesList().get(2).getOutputFiles(), outInclude2, outExclude2, false);
}

Example 8 with FileSelector

use of org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector in project scheduling by ow2-proactive.

the class StaxJobFactory method setIOFIles.

/**
 * Create the list of includes/excludes pattern for the given INPUT/OUTPUT files
 * Leave the method with the cursor at the end of 'ELEMENT_DS_INPUT/OUTPUTFILES' tag.
 *
 * @param cursorTask the streamReader with the cursor on the 'ELEMENT_DS_INPUT/OUTPUTFILES' tag.
 * @param endTag     the final tag for this tag : ELEMENT_DS_INPUTFILES or ELEMENT_DS_INPUTFILES
 * @param task       the task in which to add the input/output files selector
 * @throws JobCreationException
 */
private void setIOFIles(XMLStreamReader cursorTask, String endTag, Task task, Map<String, String> variables) throws JobCreationException {
    int i = 0;
    try {
        int eventType;
        boolean shouldContinue = true;
        while (shouldContinue && cursorTask.hasNext()) {
            eventType = cursorTask.next();
            switch(eventType) {
                case XMLEvent.START_ELEMENT:
                    String current = cursorTask.getLocalName();
                    if (XMLTags.DS_FILES.matches(current)) {
                        int attrLen = cursorTask.getAttributeCount();
                        FileSelector selector = null;
                        String accessMode = null;
                        for (i = 0; i < attrLen; i++) {
                            String attrName = cursorTask.getAttributeLocalName(i);
                            if (XMLAttributes.DS_INCLUDES.matches(attrName)) {
                                if (selector == null) {
                                    selector = new FileSelector();
                                }
                                selector.setIncludes(cursorTask.getAttributeValue(i));
                            } else if (XMLAttributes.DS_EXCLUDES.matches(attrName)) {
                                if (selector == null) {
                                    selector = new FileSelector();
                                }
                                selector.setExcludes(replace(cursorTask.getAttributeValue(i), variables));
                            } else if (XMLAttributes.DS_ACCESS_MODE.matches(attrName)) {
                                accessMode = cursorTask.getAttributeValue(i);
                            }
                            if (selector != null && accessMode != null) {
                                if (XMLTags.DS_INPUT_FILES.matches(endTag)) {
                                    task.addInputFiles(selector, InputAccessMode.getAccessMode(accessMode));
                                } else {
                                    task.addOutputFiles(selector, OutputAccessMode.getAccessMode(accessMode));
                                }
                            }
                        }
                    }
                    break;
                case XMLEvent.END_ELEMENT:
                    if (cursorTask.getLocalName().equals(endTag)) {
                        shouldContinue = false;
                    }
                    break;
            }
        }
    } catch (JobCreationException jce) {
        jce.pushTag(cursorTask.getLocalName());
        throw jce;
    } catch (Exception e) {
        String attrtmp = null;
        if (cursorTask.isStartElement() && cursorTask.getAttributeCount() > i) {
            attrtmp = cursorTask.getAttributeLocalName(i);
        }
        throw new JobCreationException(cursorTask.getLocalName(), attrtmp, e);
    }
}

Example 9 with FileSelector

use of org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector in project scheduling by ow2-proactive.

the class TaskLauncherDataSpacesTest method output_file_using_task_id_in_its_selector.

@Test
public void output_file_using_task_id_in_its_selector() throws Throwable {
    ScriptExecutableContainer executableContainer = new ScriptExecutableContainer(new TaskScript(new SimpleScript("new File('output_' + variables.get('PA_TASK_ID') + '.txt').text = 'hello'", "groovy")));
    TaskLauncherInitializer initializer = new TaskLauncherInitializer();
    initializer.setTaskId(TaskIdImpl.createTaskId(JobIdImpl.makeJobId("1000"), "job", 1000L));
    initializer.setTaskOutputFiles(singletonList(new OutputSelector(new FileSelector("output_${PA_TASK_ID}.txt"), OutputAccessMode.TransferToGlobalSpace)));
    TaskResult taskResult = runTaskLauncher(createLauncherWithInjectedMocks(initializer, taskLauncherFactory), executableContainer);
    assertTaskResultOk(taskResult);
    assertTrue(new File(taskLauncherFactory.getDataSpaces().getGlobalURI(), "output_1000.txt").exists());
}

Example 10 with FileSelector

use of org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector in project scheduling by ow2-proactive.

the class TaskLauncher method copyTaskLogsToUserSpace.

private void copyTaskLogsToUserSpace(File taskLogFile, TaskDataspaces dataspaces) {
    if (initializer.isPreciousLogs()) {
        try {
            FileSelector taskLogFileSelector = new FileSelector(taskLogFile.getName());
            taskLogFileSelector.setIncludes(new TaskLoggerRelativePathGenerator(taskId).getRelativePath());
            dataspaces.copyScratchDataToOutput(Collections.singletonList(new OutputSelector(taskLogFileSelector, OutputAccessMode.TransferToUserSpace)));
        } catch (FileSystemException e) {
            logger.warn("Cannot copy logs of task to user data spaces", e);
        }
    }
}