Examples with OutputSelector org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector used on opensource projects

Example 1 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class RestSmartProxyImpl method downloadTaskOutputFiles.

@Override
protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, String taskName, String localFolder) throws NotConnectedException, PermissionException {
    AwaitedTask atask = awaitedjob.getAwaitedTask(taskName);
    if (atask == null) {
        throw new IllegalArgumentException("The task " + taskName + " does not belong to job " + jobId + " or has already been removed");
    }
    if (atask.isTransferring()) {
        logger.warn("The task " + taskName + " of job " + jobId + " is already transferring its output");
        return;
    }
    String outputSpace = awaitedjob.getOutputSpaceURL();
    String sourceFile;
    try {
        String userSpace = getLocalUserSpace();
        if (!outputSpace.startsWith(userSpace)) {
            logger.warn("RestSmartProxy does not support data transfers outside USERSPACE.");
        }
        sourceFile = outputSpace.substring(userSpace.length() + 1);
    } catch (Throwable error) {
        throw Throwables.propagate(error);
    }
    if (awaitedjob.isIsolateTaskOutputs()) {
        sourceFile = sourceFile.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + atask.getTaskId());
    }
    List<OutputSelector> outputFileSelectors = atask.getOutputSelectors();
    List<String> includes = Lists.newArrayList();
    List<String> excludes = Lists.newArrayList();
    if (outputFileSelectors != null) {
        for (OutputSelector os : outputFileSelectors) {
            addfileSelection(os.getOutputFiles(), includes, excludes);
        }
    }
    jobTracker.setTaskTransferring(jobId, taskName, true);
    if (awaitedjob.isAutomaticTransfer()) {
        threadPool.submit(new DownloadHandler(jobId, taskName, sourceFile, includes, excludes, localFolder));
    } else {
        try {
            RemoteSource source = new RemoteSource(USER, sourceFile);
            source.setIncludes(includes);
            source.setExcludes(excludes);
            File localDir = new File(localFolder);
            LocalDestination dest = new LocalDestination(localDir);
            restDataSpaceClient.download(source, dest);
        } catch (NotConnectedException | PermissionException e) {
            logger.error(String.format("Cannot download files, jobId=%s, taskId=%s, source=%s, destination=%s", jobId, taskName, sourceFile, localFolder), e);
            throw e;
        } finally {
            jobTracker.setTaskTransferring(jobId, taskName, false);
        }
        // task is removed from the job tracker only if the transfer is successful
        jobTracker.removeAwaitedTask(jobId, taskName);
    }
}

Example 2 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class Attribute method createTaskElement.

/**
 * Creates the task element, corressponding to <define name="task">
 */
private Element createTaskElement(Document doc, Task task) {
    Element taskE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.TASK.getXMLName());
    if (task.getOnTaskErrorProperty().isSet()) {
        setAttribute(taskE, XMLAttributes.COMMON_ON_TASK_ERROR, task.getOnTaskErrorProperty().getValue().toString(), true);
    }
    if (task.getMaxNumberOfExecutionProperty().isSet()) {
        setAttribute(taskE, XMLAttributes.COMMON_MAX_NUMBER_OF_EXECUTION, Integer.toString(task.getMaxNumberOfExecution()));
    }
    setAttribute(taskE, XMLAttributes.COMMON_NAME, task.getName(), true);
    if (task.getRestartTaskOnErrorProperty().isSet()) {
        setAttribute(taskE, XMLAttributes.COMMON_RESTART_TASK_ON_ERROR, task.getRestartTaskOnError().toString());
    }
    // *** task attributes ***
    if (task.getWallTime() != 0) {
        setAttribute(taskE, XMLAttributes.TASK_WALLTIME, formatDate(task.getWallTime()));
    }
    if (task.isRunAsMe()) {
        setAttribute(taskE, XMLAttributes.TASK_RUN_AS_ME, "true");
    }
    if (task.isPreciousResult()) {
        setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_RESULT, "true");
    }
    if (task.isPreciousLogs()) {
        setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_LOGS, "true");
    }
    // <ref name="taskDescription"/>
    if (task.getDescription() != null) {
        Element descrNode = createElement(doc, XMLTags.COMMON_DESCRIPTION.getXMLName(), task.getDescription());
        taskE.appendChild(descrNode);
    }
    // <ref name="variables"/>
    if (task.getVariables() != null && !task.getVariables().isEmpty()) {
        Element variablesE = createTaskVariablesElement(doc, task.getVariables());
        taskE.appendChild(variablesE);
    }
    // <ref name="genericInformation"/>
    if ((task.getGenericInformation() != null) && (task.getGenericInformation().size() > 0)) {
        Element genericInfoE = createGenericInformation(doc, task.getGenericInformation());
        taskE.appendChild(genericInfoE);
    }
    // <ref name="depends"/>
    List<Task> dependencies = task.getDependencesList();
    if ((dependencies != null) && (dependencies.size() > 0)) {
        Element dependsE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.TASK_DEPENDENCES.getXMLName());
        for (Task dep : dependencies) {
            Element dependsTask = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.TASK_DEPENDENCES_TASK.getXMLName());
            setAttribute(dependsTask, XMLAttributes.TASK_DEPENDS_REF, dep.getName(), true);
            dependsE.appendChild(dependsTask);
        }
        taskE.appendChild(dependsE);
    }
    // if has dependencies
    // <ref name="inputFiles"/>
    List<InputSelector> inputFiles = task.getInputFilesList();
    if (inputFiles != null) {
        Element inputFilesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_INPUT_FILES.getXMLName());
        for (InputSelector inputSelector : inputFiles) {
            FileSelector fs = inputSelector.getInputFiles();
            Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_FILES.getXMLName());
            // pattern
            if (!fs.getIncludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true);
            if (!fs.getExcludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true);
            if (inputSelector.getMode() != null) {
                setAttribute(filesE, XMLAttributes.DS_ACCESS_MODE, inputSelector.getMode().toString(), true);
            }
            inputFilesE.appendChild(filesE);
        }
        taskE.appendChild(inputFilesE);
    }
    // <ref name="parallel"/>
    Element parallelEnvE = createParallelEnvironment(doc, task);
    if (parallelEnvE != null)
        taskE.appendChild(parallelEnvE);
    // <ref name="selection"/>
    List<SelectionScript> selectionScripts = task.getSelectionScripts();
    if (selectionScripts != null && selectionScripts.size() > 0) {
        Element selectionE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_SELECTION.getXMLName());
        for (SelectionScript selectionScript : selectionScripts) {
            Element scriptE = createScriptElement(doc, selectionScript);
            selectionE.appendChild(scriptE);
        }
        taskE.appendChild(selectionE);
    }
    // <ref name="forkEnvironment"/>
    if (task.getForkEnvironment() != null) {
        Element forkEnvE = createForkEnvironmentElement(doc, task.getForkEnvironment());
        taskE.appendChild(forkEnvE);
    }
    // <ref name="pre"/>
    Script preScript = task.getPreScript();
    if (preScript != null) {
        Element preE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_PRE.getXMLName());
        Element scriptE = createScriptElement(doc, preScript);
        preE.appendChild(scriptE);
        taskE.appendChild(preE);
    }
    // <ref name="executable"/>
    Element executableE = null;
    if (task instanceof JavaTask) {
        executableE = createJavaExecutableElement(doc, (JavaTask) task);
    } else if (task instanceof NativeTask) {
        executableE = createNativeExecutableElement(doc, (NativeTask) task);
    } else if (task instanceof ScriptTask) {
        executableE = createScriptExecutableElement(doc, (ScriptTask) task);
    }
    taskE.appendChild(executableE);
    // <ref name="flow"/>
    Element controlFlowE = createFlowControlElement(doc, task);
    if (controlFlowE != null)
        taskE.appendChild(controlFlowE);
    // <ref name="post"/>
    Script postScript = task.getPostScript();
    if (postScript != null) {
        Element postE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_POST.getXMLName());
        Element scriptE = createScriptElement(doc, postScript);
        postE.appendChild(scriptE);
        taskE.appendChild(postE);
    }
    // <ref name="cleaning"/>
    Script cleanScript = task.getCleaningScript();
    if (cleanScript != null) {
        Element cleanE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.SCRIPT_CLEANING.getXMLName());
        Element scriptE = createScriptElement(doc, cleanScript);
        cleanE.appendChild(scriptE);
        taskE.appendChild(cleanE);
    }
    // <ref name="outputFiles"/>
    List<OutputSelector> outputFiles = task.getOutputFilesList();
    if (outputFiles != null) {
        Element outputFilesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_OUTPUT_FILES.getXMLName());
        for (OutputSelector outputSelector : outputFiles) {
            FileSelector fs = outputSelector.getOutputFiles();
            Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.getNamespace(), XMLTags.DS_FILES.getXMLName());
            // pattern
            if (!fs.getIncludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true);
            if (!fs.getExcludes().isEmpty())
                setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true);
            if (outputSelector.getMode() != null) {
                setAttribute(filesE, XMLAttributes.DS_ACCESS_MODE, outputSelector.getMode().toString(), true);
            }
            outputFilesE.appendChild(filesE);
        }
        taskE.appendChild(outputFilesE);
    }
    return taskE;
}

Example 3 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method copyScratchDataToOutput.

@Override
public void copyScratchDataToOutput(List<OutputSelector> outputSelectors) throws FileSystemException {
    try {
        if (outputSelectors == null) {
            logger.debug("Output selector is empty, no file to copy");
            return;
        }
        SCRATCH.refresh();
        checkOutputSpacesConfigured(outputSelectors);
        ArrayList<DataSpacesFileObject> results = new ArrayList<>();
        FileSystemException toBeThrown = null;
        for (OutputSelector os : outputSelectors) {
            org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector selector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
            selector.setIncludes(os.getOutputFiles().getIncludes());
            selector.setExcludes(os.getOutputFiles().getExcludes());
            switch(os.getMode()) {
                case TransferToOutputSpace:
                    if (OUTPUT != null) {
                        toBeThrown = copyScratchDataToOutput(OUTPUT, "OUTPUT", os, selector, results);
                    }
                    break;
                case TransferToGlobalSpace:
                    if (GLOBAL != null) {
                        toBeThrown = copyScratchDataToOutput(GLOBAL, "GLOBAL", os, selector, results);
                    }
                    break;
                case TransferToUserSpace:
                    if (USER != null) {
                        toBeThrown = copyScratchDataToOutput(USER, "USER", os, selector, results);
                        break;
                    }
                case none:
                    break;
            }
            results.clear();
        }
        if (toBeThrown != null) {
            throw toBeThrown;
        }
    } finally {
        // display dataspaces error and warns if any
        displayDataspacesStatus();
    }
}

Example 4 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class TaskLauncherDataSpacesTest method output_file_using_variable_in_its_selector.

@Test
public void output_file_using_variable_in_its_selector() throws Throwable {
    ScriptExecutableContainer createAFileAndWriteVariable = new ScriptExecutableContainer(new TaskScript(new SimpleScript("new File('output_foo_bar.txt').text = 'hello'; variables.put('aVar', 'foo')", "groovy")));
    TaskLauncherInitializer copyOutputFile = new TaskLauncherInitializer();
    copyOutputFile.setTaskId(TaskIdImpl.createTaskId(JobIdImpl.makeJobId("1000"), "job", 1000L));
    copyOutputFile.setTaskOutputFiles(singletonList(new OutputSelector(new FileSelector("output_${aVar}_${aResultVar}.txt"), OutputAccessMode.TransferToGlobalSpace)));
    TaskResultImpl previousTaskResult = taskResult(Collections.<String, Serializable>singletonMap("aResultVar", "bar"));
    TaskResult taskResult = runTaskLauncher(createLauncherWithInjectedMocks(copyOutputFile, taskLauncherFactory), createAFileAndWriteVariable, previousTaskResult);
    assertTaskResultOk(taskResult);
    assertTrue(new File(taskLauncherFactory.getDataSpaces().getGlobalURI(), "output_foo_bar.txt").exists());
}

Example 5 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class TaskLauncherInitializerTest method output_files_can_be_filtered.

@Test
public void output_files_can_be_filtered() throws Exception {
    initializer.setTaskOutputFiles(outputSelectors("${TEST}/a", "b"));
    initializer.getTaskOutputFiles().get(0).getOutputFiles().setExcludes("$TEST/excluded");
    List<OutputSelector> filteredOutputFiles = initializer.getFilteredOutputFiles(Collections.<String, Serializable>singletonMap("TEST", "folder"));
    OutputSelector selector = filteredOutputFiles.get(0);
    assertEquals(ImmutableSet.of("folder/a", "b"), selector.getOutputFiles().getIncludes());
    assertEquals(ImmutableSet.of("folder/excluded"), selector.getOutputFiles().getExcludes());
}