Examples with FileSelector org.apache.commons.vfs2.FileSelector used on opensource projects

Example 1 with FileSelector

use of org.apache.commons.vfs2.FileSelector in project scheduling by ow2-proactive.

the class SmartProxyImpl method downloadTaskOutputFiles.

@Override
protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, String t_name, String localFolder) throws Exception {
    AwaitedTask atask = awaitedjob.getAwaitedTask(t_name);
    if (atask == null) {
        throw new IllegalArgumentException("The task " + t_name + " does not belong to job " + jobId + " or has already been removed");
    }
    if (atask.isTransferring()) {
        log.warn("The task " + t_name + " of job " + jobId + " is already transferring its output");
        return;
    }
    String pull_URL = awaitedjob.getPullURL();
    if (awaitedjob.isIsolateTaskOutputs()) {
        pull_URL = pull_URL.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + atask.getTaskId());
    }
    FileObject remotePullFolderFO;
    FileObject localfolderFO;
    try {
        remotePullFolderFO = jobTracker.resolveFile(pull_URL);
        localfolderFO = jobTracker.resolveFile(localFolder);
    } catch (FileSystemException e) {
        log.error("Could not retrieve data for job " + jobId, e);
        throw new IllegalStateException("Could not retrieve data for job " + jobId, e);
    }
    String sourceUrl = remotePullFolderFO.getURL().toString();
    String destUrl = localfolderFO.getURL().toString();
    org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fileSelector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
    List<OutputSelector> ouputFileSelectors = atask.getOutputSelectors();
    for (OutputSelector os : ouputFileSelectors) {
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fs = os.getOutputFiles();
        if (!fs.getIncludes().isEmpty()) {
            fileSelector.addIncludes(fs.getIncludes());
        }
        if (!fs.getExcludes().isEmpty()) {
            fileSelector.addExcludes(fs.getExcludes());
        }
    }
    if (log.isDebugEnabled()) {
        log.debug("Looking at files in " + sourceUrl + " with " + fileSelector.getIncludes() + "-" + fileSelector.getExcludes());
        boolean goon = true;
        int cpt = 0;
        FileObject[] fos = null;
        while (goon) {
            fos = remotePullFolderFO.findFiles(fileSelector);
            goon = cpt < 50 && (fos == null || fos.length == 0);
            cpt++;
            if (goon) {
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        }
        if (fos != null && fos.length > 0) {
            for (FileObject fo : fos) {
                log.debug("Found " + fo.getName());
            }
        } else {
            log.warn("Couldn't find " + fileSelector.getIncludes() + "-" + fileSelector.getExcludes() + " in " + sourceUrl);
        }
    }
    if (awaitedjob.isAutomaticTransfer()) {
        DataTransferProcessor dtp = new DataTransferProcessor(remotePullFolderFO, localfolderFO, jobId, t_name, fileSelector);
        jobTracker.setTaskTransferring(jobId, t_name, true);
        threadPool.submit((Runnable) dtp);
    } else {
        log.debug("Copying files from " + sourceUrl + " to " + destUrl);
        try {
            localfolderFO.copyFrom(remotePullFolderFO, fileSelector);
        } catch (FileSystemException e) {
            log.error(e);
            throw e;
        } finally {
            jobTracker.setTaskTransferring(jobId, t_name, false);
        }
        // task is removed from the job tracker only if the transfer is successful
        jobTracker.removeAwaitedTask(jobId, t_name);
        log.debug("Finished copying files from " + sourceUrl + " to " + destUrl);
    // ok we can remove the task
    }
}

Example 2 with FileSelector

use of org.apache.commons.vfs2.FileSelector in project scheduling by ow2-proactive.

the class SmartProxyImpl method uploadInputfiles.

@Override
public boolean uploadInputfiles(TaskFlowJob job, String localInputFolderPath) throws Exception {
    String push_URL = job.getGenericInformation().get(GENERIC_INFO_PUSH_URL_PROPERTY_NAME);
    if ((push_URL == null) || (push_URL.trim().equals(""))) {
        return false;
    }
    // push inputData
    // TODO - if the copy fails, try to remove the files from the remote
    // folder before throwing an exception
    FileObject remoteFolder = jobTracker.resolveFile(push_URL);
    FileObject localfolder = jobTracker.resolveFile(localInputFolderPath);
    String jname = job.getName();
    log.debug("Pushing files for job " + jname + " from " + localfolder + " to " + remoteFolder);
    TaskFlowJob tfj = job;
    ArrayList<Task> tasks = tfj.getTasks();
    List<DataTransferProcessor> transferCallables = new ArrayList<>(tasks.size());
    for (Task t : tasks) {
        log.debug("Pushing files for task " + t.getName());
        List<InputSelector> inputFileSelectors = t.getInputFilesList();
        // create the selector
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fileSelector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
        for (InputSelector is : inputFileSelectors) {
            org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fs = is.getInputFiles();
            if (!fs.getIncludes().isEmpty()) {
                fileSelector.addIncludes(fs.getIncludes());
            }
            if (!fs.getExcludes().isEmpty()) {
                fileSelector.addExcludes(fs.getExcludes());
            }
        // We should check if a pattern exist in both includes and
        // excludes. But that would be a user mistake.
        }
        DataTransferProcessor dtp = new DataTransferProcessor(localfolder, remoteFolder, tfj.getName(), t.getName(), fileSelector);
        transferCallables.add(dtp);
    }
    List<Future<Boolean>> futures;
    try {
        futures = threadPool.invokeAll(transferCallables);
    } catch (InterruptedException e) {
        log.error("Interrupted while transferring files of job " + jname, e);
        throw new RuntimeException(e);
    }
    for (int i = 0; i < futures.size(); i++) {
        Future<Boolean> answer = futures.get(i);
        String tname = tfj.getTasks().get(i).getName();
        try {
            if (!answer.get()) {
                // this should not happen
                throw new RuntimeException("Files of task " + tname + " for job " + jname + " could not be transferred");
            }
        } catch (InterruptedException e) {
            log.error("Interrupted while transferring files of task " + tname + " for job " + jname, e);
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            log.error("Exception occured while transferring files of task " + tname + " for job " + jname, e);
            throw new RuntimeException(e);
        }
    }
    log.debug("Finished push operation from " + localfolder + " to " + remoteFolder);
    return true;
}