Examples with OutputSelector org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector used on opensource projects

Example 6 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class SmartProxyImpl method downloadTaskOutputFiles.

@Override
protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, String t_name, String localFolder) throws Exception {
    AwaitedTask atask = awaitedjob.getAwaitedTask(t_name);
    if (atask == null) {
        throw new IllegalArgumentException("The task " + t_name + " does not belong to job " + jobId + " or has already been removed");
    }
    if (atask.isTransferring()) {
        log.warn("The task " + t_name + " of job " + jobId + " is already transferring its output");
        return;
    }
    String pull_URL = awaitedjob.getPullURL();
    if (awaitedjob.isIsolateTaskOutputs()) {
        pull_URL = pull_URL.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + atask.getTaskId());
    }
    FileObject remotePullFolderFO;
    FileObject localfolderFO;
    try {
        remotePullFolderFO = jobTracker.resolveFile(pull_URL);
        localfolderFO = jobTracker.resolveFile(localFolder);
    } catch (FileSystemException e) {
        log.error("Could not retrieve data for job " + jobId, e);
        throw new IllegalStateException("Could not retrieve data for job " + jobId, e);
    }
    String sourceUrl = remotePullFolderFO.getURL().toString();
    String destUrl = localfolderFO.getURL().toString();
    org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fileSelector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
    List<OutputSelector> ouputFileSelectors = atask.getOutputSelectors();
    for (OutputSelector os : ouputFileSelectors) {
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fs = os.getOutputFiles();
        if (!fs.getIncludes().isEmpty()) {
            fileSelector.addIncludes(fs.getIncludes());
        }
        if (!fs.getExcludes().isEmpty()) {
            fileSelector.addExcludes(fs.getExcludes());
        }
    }
    if (log.isDebugEnabled()) {
        log.debug("Looking at files in " + sourceUrl + " with " + fileSelector.getIncludes() + "-" + fileSelector.getExcludes());
        boolean goon = true;
        int cpt = 0;
        FileObject[] fos = null;
        while (goon) {
            fos = remotePullFolderFO.findFiles(fileSelector);
            goon = cpt < 50 && (fos == null || fos.length == 0);
            cpt++;
            if (goon) {
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        }
        if (fos != null && fos.length > 0) {
            for (FileObject fo : fos) {
                log.debug("Found " + fo.getName());
            }
        } else {
            log.warn("Couldn't find " + fileSelector.getIncludes() + "-" + fileSelector.getExcludes() + " in " + sourceUrl);
        }
    }
    if (awaitedjob.isAutomaticTransfer()) {
        DataTransferProcessor dtp = new DataTransferProcessor(remotePullFolderFO, localfolderFO, jobId, t_name, fileSelector);
        jobTracker.setTaskTransferring(jobId, t_name, true);
        threadPool.submit((Runnable) dtp);
    } else {
        log.debug("Copying files from " + sourceUrl + " to " + destUrl);
        try {
            localfolderFO.copyFrom(remotePullFolderFO, fileSelector);
        } catch (FileSystemException e) {
            log.error(e);
            throw e;
        } finally {
            jobTracker.setTaskTransferring(jobId, t_name, false);
        }
        // task is removed from the job tracker only if the transfer is successful
        jobTracker.removeAwaitedTask(jobId, t_name);
        log.debug("Finished copying files from " + sourceUrl + " to " + destUrl);
    // ok we can remove the task
    }
}

Example 7 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class JobComparator method isEqualOutputFiles.

/**
 * Compares the element in the 2 lists in the exact order they appear in the
 * lists FIXME: bad object design in the data space layer provides us to
 * unify the similar code in this method and isEqualInputFiles
 */
private boolean isEqualOutputFiles(List<OutputSelector> l1, List<OutputSelector> l2) {
    if ((l1 == null) && (l2 == null))
        return true;
    if ((l1 == null) ^ (l2 == null)) {
        stack.push("One of 2 values is null");
        return false;
    }
    if (l1.size() != l2.size())
        return false;
    for (OutputSelector os1 : l1) {
        boolean found = false;
        for (int i = 0; i < l2.size(); i++) {
            OutputSelector os2 = l2.get(i);
            if (!os1.getMode().equals(os2.getMode())) {
                continue;
            }
            Set<String> i1 = os1.getOutputFiles().getIncludes();
            Set<String> i2 = os2.getOutputFiles().getIncludes();
            if (!i1.equals(i2)) {
                continue;
            }
            Set<String> e1 = os1.getOutputFiles().getExcludes();
            Set<String> e2 = os2.getOutputFiles().getExcludes();
            if (!e1.equals(e2)) {
                continue;
            }
            found = true;
            break;
        }
        if (!found) {
            return false;
        }
    }
    return true;
}

Example 8 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class TaskLauncherDataSpacesTest method output_file_using_task_id_in_its_selector.

@Test
public void output_file_using_task_id_in_its_selector() throws Throwable {
    ScriptExecutableContainer executableContainer = new ScriptExecutableContainer(new TaskScript(new SimpleScript("new File('output_' + variables.get('PA_TASK_ID') + '.txt').text = 'hello'", "groovy")));
    TaskLauncherInitializer initializer = new TaskLauncherInitializer();
    initializer.setTaskId(TaskIdImpl.createTaskId(JobIdImpl.makeJobId("1000"), "job", 1000L));
    initializer.setTaskOutputFiles(singletonList(new OutputSelector(new FileSelector("output_${PA_TASK_ID}.txt"), OutputAccessMode.TransferToGlobalSpace)));
    TaskResult taskResult = runTaskLauncher(createLauncherWithInjectedMocks(initializer, taskLauncherFactory), executableContainer);
    assertTaskResultOk(taskResult);
    assertTrue(new File(taskLauncherFactory.getDataSpaces().getGlobalURI(), "output_1000.txt").exists());
}

Example 9 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class TaskLauncherInitializerTest method output_empty_filters.

@Test
public void output_empty_filters() throws Exception {
    initializer.setTaskOutputFiles(outputSelectors("$TEST/a", "b"));
    List<OutputSelector> filteredOutputFiles = initializer.getFilteredOutputFiles(Collections.<String, Serializable>emptyMap());
    OutputSelector selector = filteredOutputFiles.get(0);
    assertEquals(ImmutableSet.of("$TEST/a", "b"), selector.getOutputFiles().getIncludes());
}

Example 10 with OutputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.OutputSelector in project scheduling by ow2-proactive.

the class TaskLauncherInitializerTest method output_null_filters.

@Test
public void output_null_filters() throws Exception {
    initializer.setTaskOutputFiles(outputSelectors("$TEST/a", "b"));
    List<OutputSelector> filteredOutputFiles = initializer.getFilteredOutputFiles(null);
    OutputSelector selector = filteredOutputFiles.get(0);
    assertEquals(ImmutableSet.of("$TEST/a", "b"), selector.getOutputFiles().getIncludes());
}