Examples with InputSelector org.ow2.proactive.scheduler.common.task.dataspaces.InputSelector used on opensource projects

Example 11 with InputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.InputSelector in project scheduling by ow2-proactive.

the class RestSmartProxyImpl method uploadInputfiles.

/**
 * @throws NotConnectedException
 * @throws PermissionException
 * @see AbstractSmartProxy#uploadInputfiles(TaskFlowJob, String)
 */
@Override
public boolean uploadInputfiles(TaskFlowJob job, String localInputFolderPath) throws NotConnectedException, PermissionException {
    String userSpace = getLocalUserSpace();
    String inputSpace = job.getInputSpace();
    if (!inputSpace.startsWith(userSpace)) {
        // NOTE: only works for USERSPACE urls
        logger.warn("RestSmartProxy does not support data transfers outside USERSPACE.");
        return false;
    }
    String remotePath = inputSpace.substring(userSpace.length() + (userSpace.endsWith("/") ? 0 : 1));
    String jname = job.getName();
    logger.debug("Pushing files for job " + jname + " from " + localInputFolderPath + " to " + remotePath);
    TaskFlowJob tfj = job;
    for (Task t : tfj.getTasks()) {
        logger.debug("Pushing files for task " + t.getName());
        List<String> includes = Lists.newArrayList();
        List<String> excludes = Lists.newArrayList();
        List<InputSelector> inputFilesList = t.getInputFilesList();
        if (inputFilesList != null) {
            for (InputSelector is : inputFilesList) {
                addfileSelection(is.getInputFiles(), includes, excludes);
            }
        }
        LocalDirSource source = new LocalDirSource(localInputFolderPath);
        source.setIncludes(includes);
        source.setExcludes(excludes);
        RemoteDestination dest = new RemoteDestination(USER, remotePath);
        restDataSpaceClient.upload(source, dest);
    }
    logger.debug("Finished push operation from " + localInputFolderPath + " to " + remotePath);
    return true;
}

Example 12 with InputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.InputSelector in project scheduling by ow2-proactive.

the class SmartProxyImpl method uploadInputfiles.

@Override
public boolean uploadInputfiles(TaskFlowJob job, String localInputFolderPath) throws Exception {
    String push_URL = job.getGenericInformation().get(GENERIC_INFO_PUSH_URL_PROPERTY_NAME);
    if ((push_URL == null) || (push_URL.trim().equals(""))) {
        return false;
    }
    // push inputData
    // TODO - if the copy fails, try to remove the files from the remote
    // folder before throwing an exception
    FileObject remoteFolder = jobTracker.resolveFile(push_URL);
    FileObject localfolder = jobTracker.resolveFile(localInputFolderPath);
    String jname = job.getName();
    log.debug("Pushing files for job " + jname + " from " + localfolder + " to " + remoteFolder);
    TaskFlowJob tfj = job;
    ArrayList<Task> tasks = tfj.getTasks();
    List<DataTransferProcessor> transferCallables = new ArrayList<>(tasks.size());
    for (Task t : tasks) {
        log.debug("Pushing files for task " + t.getName());
        List<InputSelector> inputFileSelectors = t.getInputFilesList();
        // create the selector
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fileSelector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
        for (InputSelector is : inputFileSelectors) {
            org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fs = is.getInputFiles();
            if (!fs.getIncludes().isEmpty()) {
                fileSelector.addIncludes(fs.getIncludes());
            }
            if (!fs.getExcludes().isEmpty()) {
                fileSelector.addExcludes(fs.getExcludes());
            }
        // We should check if a pattern exist in both includes and
        // excludes. But that would be a user mistake.
        }
        DataTransferProcessor dtp = new DataTransferProcessor(localfolder, remoteFolder, tfj.getName(), t.getName(), fileSelector);
        transferCallables.add(dtp);
    }
    List<Future<Boolean>> futures;
    try {
        futures = threadPool.invokeAll(transferCallables);
    } catch (InterruptedException e) {
        log.error("Interrupted while transferring files of job " + jname, e);
        throw new RuntimeException(e);
    }
    for (int i = 0; i < futures.size(); i++) {
        Future<Boolean> answer = futures.get(i);
        String tname = tfj.getTasks().get(i).getName();
        try {
            if (!answer.get()) {
                // this should not happen
                throw new RuntimeException("Files of task " + tname + " for job " + jname + " could not be transferred");
            }
        } catch (InterruptedException e) {
            log.error("Interrupted while transferring files of task " + tname + " for job " + jname, e);
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            log.error("Exception occured while transferring files of task " + tname + " for job " + jname, e);
            throw new RuntimeException(e);
        }
    }
    log.debug("Finished push operation from " + localfolder + " to " + remoteFolder);
    return true;
}

Example 13 with InputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.InputSelector in project scheduling by ow2-proactive.

the class SchedulerDBManager method toInternalTasks.

private Collection<InternalTask> toInternalTasks(boolean loadFullState, InternalJob internalJob, List<TaskData> taskRuntimeDataList) {
    Map<DBTaskId, InternalTask> tasks = new HashMap<>(taskRuntimeDataList.size());
    try {
        for (TaskData taskData : taskRuntimeDataList) {
            InternalTask internalTask = taskData.toInternalTask(internalJob, loadFullState);
            if (loadFullState) {
                internalTask.setParallelEnvironment(taskData.getParallelEnvironment());
                internalTask.setGenericInformation(taskData.getGenericInformation());
                for (SelectionScriptData scriptData : taskData.getSelectionScripts()) {
                    internalTask.addSelectionScript(scriptData.createSelectionScript());
                }
                if (taskData.getCleanScript() != null) {
                    internalTask.setCleaningScript(taskData.getCleanScript().createSimpleScript());
                }
                if (taskData.getPreScript() != null) {
                    internalTask.setPreScript(taskData.getPreScript().createSimpleScript());
                }
                if (taskData.getPostScript() != null) {
                    internalTask.setPostScript(taskData.getPostScript().createSimpleScript());
                }
                if (taskData.getFlowScript() != null) {
                    internalTask.setFlowScript(taskData.getFlowScript().createFlowScript());
                }
                for (SelectorData selectorData : taskData.getDataspaceSelectors()) {
                    if (selectorData.isInput()) {
                        InputSelector selector = selectorData.createInputSelector();
                        internalTask.addInputFiles(selector.getInputFiles(), selector.getMode());
                    } else {
                        OutputSelector selector = selectorData.createOutputSelector();
                        internalTask.addOutputFiles(selector.getOutputFiles(), selector.getMode());
                    }
                }
            }
            tasks.put(taskData.getId(), internalTask);
        }
    } catch (InvalidScriptException e) {
        throw new DatabaseManagerException("Failed to initialize loaded script", e);
    }
    for (TaskData taskData : taskRuntimeDataList) {
        InternalTask internalTask = tasks.get(taskData.getId());
        if (!taskData.getDependentTasks().isEmpty()) {
            for (DBTaskId dependent : taskData.getDependentTasks()) {
                internalTask.addDependence(tasks.get(dependent));
            }
        }
        if (loadFullState) {
            if (taskData.getIfBranch() != null) {
                internalTask.setIfBranch(tasks.get(taskData.getIfBranch().getId()));
            }
            if (!taskData.getJoinedBranches().isEmpty()) {
                List<InternalTask> branches = new ArrayList<>(taskData.getJoinedBranches().size());
                for (DBTaskId joinedBranch : taskData.getJoinedBranches()) {
                    branches.add(tasks.get(joinedBranch));
                }
                internalTask.setJoinedBranches(branches);
            }
            internalTask.setName(internalTask.getName());
        }
    }
    return tasks.values();
}

Example 14 with InputSelector

use of org.ow2.proactive.scheduler.common.task.dataspaces.InputSelector in project scheduling by ow2-proactive.

the class TaskData method createTaskData.

static TaskData createTaskData(JobData jobRuntimeData, InternalScriptTask task) {
    TaskData taskData = new TaskData();
    TaskData.DBTaskId taskId = new DBTaskId();
    taskId.setJobId(jobRuntimeData.getId());
    taskId.setTaskId(task.getTaskInfo().getTaskId().longValue());
    taskData.setId(taskId);
    taskData.setDescription(task.getDescription());
    taskData.setTag(task.getTag());
    taskData.setParallelEnvironment(task.getParallelEnvironment());
    taskData.setFlowBlock(task.getFlowBlock());
    taskData.setRestartMode(task.getRestartTaskOnError());
    taskData.setPreciousLogs(task.isPreciousLogs());
    taskData.setPreciousResult(task.isPreciousResult());
    taskData.setRunAsMe(task.isRunAsMe());
    taskData.setWallTime(task.getWallTime());
    taskData.setOnTaskErrorString(task.getOnTaskErrorProperty().getValue());
    taskData.setMaxNumberOfExecution(task.getMaxNumberOfExecution());
    taskData.setJobData(jobRuntimeData);
    taskData.setNumberOfExecutionOnFailureLeft(PASchedulerProperties.NUMBER_OF_EXECUTION_ON_FAILURE.getValueAsInt());
    taskData.setNumberOfExecutionLeft(task.getMaxNumberOfExecution());
    taskData.setGenericInformation(task.getGenericInformation());
    HashMap<String, TaskDataVariable> variables = new HashMap<>();
    for (Map.Entry<String, TaskVariable> entry : task.getVariables().entrySet()) {
        variables.put(entry.getKey(), TaskDataVariable.create(entry.getKey(), entry.getValue(), taskData));
    }
    taskData.setVariables(variables);
    // set the scheduledTime if the START_AT property exists
    Map<String, String> genericInfos = taskData.getGenericInformation();
    if (genericInfos != null && genericInfos.containsKey(CommonAttribute.GENERIC_INFO_START_AT_KEY)) {
        long scheduledTime = ISO8601DateUtil.toDate(genericInfos.get(CommonAttribute.GENERIC_INFO_START_AT_KEY)).getTime();
        taskData.setScheduledTime(scheduledTime);
        task.setScheduledTime(scheduledTime);
    }
    taskData.updateMutableAttributes(task);
    if (task.getSelectionScripts() != null) {
        List<SelectionScriptData> scripts = new ArrayList<>(task.getSelectionScripts().size());
        for (SelectionScript selectionScript : task.getSelectionScripts()) {
            scripts.add(SelectionScriptData.createForSelectionScript(selectionScript, taskData));
        }
        taskData.setSelectionScripts(scripts);
    }
    if (task.getExecutableContainer() != null) {
        taskData.setScript(ScriptData.createForScript(((ScriptExecutableContainer) task.getExecutableContainer()).getScript(), taskData));
    }
    if (task.getPreScript() != null) {
        taskData.setPreScript(ScriptData.createForScript(task.getPreScript(), taskData));
    }
    if (task.getPostScript() != null) {
        taskData.setPostScript(ScriptData.createForScript(task.getPostScript(), taskData));
    }
    if (task.getCleaningScript() != null) {
        taskData.setCleanScript(ScriptData.createForScript(task.getCleaningScript(), taskData));
    }
    if (task.getFlowScript() != null) {
        taskData.setFlowScript(ScriptData.createForFlowScript(task.getFlowScript(), taskData));
    }
    List<SelectorData> selectorsData = new ArrayList<>();
    if (task.getInputFilesList() != null) {
        for (InputSelector selector : task.getInputFilesList()) {
            selectorsData.add(SelectorData.createForInputSelector(selector, taskData));
        }
    }
    if (task.getOutputFilesList() != null) {
        for (OutputSelector selector : task.getOutputFilesList()) {
            selectorsData.add(SelectorData.createForOutputSelector(selector, taskData));
        }
    }
    taskData.setDataspaceSelectors(selectorsData);
    ForkEnvironment forkEnvironment = task.getForkEnvironment();
    if (forkEnvironment != null) {
        taskData.setAdditionalClasspath(forkEnvironment.getAdditionalClasspath());
        taskData.setJavaHome(forkEnvironment.getJavaHome());
        taskData.setJvmArguments(forkEnvironment.getJVMArguments());
        taskData.setWorkingDir(forkEnvironment.getWorkingDir());
        if (forkEnvironment.getEnvScript() != null) {
            taskData.setEnvScript(ScriptData.createForScript(forkEnvironment.getEnvScript(), taskData));
        }
        Map<String, String> systemEnvironment = forkEnvironment.getSystemEnvironment();
        if (systemEnvironment != null) {
            List<EnvironmentModifierData> envModifiers = new ArrayList<>(systemEnvironment.size());
            for (Map.Entry<String, String> entry : systemEnvironment.entrySet()) {
                envModifiers.add(EnvironmentModifierData.create(new PropertyModifier(entry.getKey(), entry.getValue()), taskData));
            }
            taskData.setEnvModifiers(envModifiers);
        }
    }
    taskData.initTaskType(task);
    return taskData;
}