Examples with FileObject org.apache.commons.vfs2.FileObject used on opensource projects

Example 36 with FileObject

use of org.apache.commons.vfs2.FileObject in project scheduling by ow2-proactive.

the class VFSZipper method zip.

public static void zip(FileObject root, List<FileObject> files, OutputStream out) throws IOException {
    String basePath = root.getName().getPath();
    Closer closer = Closer.create();
    try {
        ZipOutputStream zos = new ZipOutputStream(out);
        closer.register(zos);
        for (FileObject fileToCopy : files) {
            ZipEntry zipEntry = zipEntry(basePath, fileToCopy);
            zos.putNextEntry(zipEntry);
            copyFileContents(fileToCopy, zos);
            zos.flush();
            zos.closeEntry();
        }
    } catch (IOException e) {
        throw closer.rethrow(e);
    } finally {
        closer.close();
    }
}

Example 37 with FileObject

use of org.apache.commons.vfs2.FileObject in project scheduling by ow2-proactive.

the class SmartProxyImpl method removeJobIO.

@Override
protected void removeJobIO(Job job, String pushURL, String pullURL, String newFolderName) {
    pushURL = pushURL + "/" + newFolderName;
    FileObject fo;
    try {
        fo = jobTracker.resolveFile(pushURL);
        fo.delete(Selectors.SELECT_ALL);
        fo.delete();
    } catch (Exception e) {
        log.error("Error in removeJobIO push for job " + job.getName(), e);
    }
    pullURL = pullURL + "/" + newFolderName;
    try {
        fo = jobTracker.resolveFile(pullURL);
        fo.delete(Selectors.SELECT_ALL);
        fo.delete();
    } catch (Exception e) {
        log.error("Error in removeJobIO pull for job " + job.getName(), e);
    }
}

Example 38 with FileObject

use of org.apache.commons.vfs2.FileObject in project scheduling by ow2-proactive.

the class SmartProxyImpl method uploadInputfiles.

@Override
public boolean uploadInputfiles(TaskFlowJob job, String localInputFolderPath) throws Exception {
    String push_URL = job.getGenericInformation().get(GENERIC_INFO_PUSH_URL_PROPERTY_NAME);
    if ((push_URL == null) || (push_URL.trim().equals(""))) {
        return false;
    }
    // push inputData
    // TODO - if the copy fails, try to remove the files from the remote
    // folder before throwing an exception
    FileObject remoteFolder = jobTracker.resolveFile(push_URL);
    FileObject localfolder = jobTracker.resolveFile(localInputFolderPath);
    String jname = job.getName();
    log.debug("Pushing files for job " + jname + " from " + localfolder + " to " + remoteFolder);
    TaskFlowJob tfj = job;
    ArrayList<Task> tasks = tfj.getTasks();
    List<DataTransferProcessor> transferCallables = new ArrayList<>(tasks.size());
    for (Task t : tasks) {
        log.debug("Pushing files for task " + t.getName());
        List<InputSelector> inputFileSelectors = t.getInputFilesList();
        // create the selector
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fileSelector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
        for (InputSelector is : inputFileSelectors) {
            org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector fs = is.getInputFiles();
            if (!fs.getIncludes().isEmpty()) {
                fileSelector.addIncludes(fs.getIncludes());
            }
            if (!fs.getExcludes().isEmpty()) {
                fileSelector.addExcludes(fs.getExcludes());
            }
        // We should check if a pattern exist in both includes and
        // excludes. But that would be a user mistake.
        }
        DataTransferProcessor dtp = new DataTransferProcessor(localfolder, remoteFolder, tfj.getName(), t.getName(), fileSelector);
        transferCallables.add(dtp);
    }
    List<Future<Boolean>> futures;
    try {
        futures = threadPool.invokeAll(transferCallables);
    } catch (InterruptedException e) {
        log.error("Interrupted while transferring files of job " + jname, e);
        throw new RuntimeException(e);
    }
    for (int i = 0; i < futures.size(); i++) {
        Future<Boolean> answer = futures.get(i);
        String tname = tfj.getTasks().get(i).getName();
        try {
            if (!answer.get()) {
                // this should not happen
                throw new RuntimeException("Files of task " + tname + " for job " + jname + " could not be transferred");
            }
        } catch (InterruptedException e) {
            log.error("Interrupted while transferring files of task " + tname + " for job " + jname, e);
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            log.error("Exception occured while transferring files of task " + tname + " for job " + jname, e);
            throw new RuntimeException(e);
        }
    }
    log.debug("Finished push operation from " + localfolder + " to " + remoteFolder);
    return true;
}

Example 39 with FileObject

use of org.apache.commons.vfs2.FileObject in project scheduling by ow2-proactive.

the class JobTrackerImpl method removeAwaitedJob.

/**
 * Removes from the proxy knowledge all info related with the given job.
 * This will also delete every folder created by the job in the shared input and output spaces
 *
 * @param id jobID
 */
public void removeAwaitedJob(String id) {
    AwaitedJob aj = jobDatabase.getAwaitedJob(id);
    if (aj == null) {
        logger.warn("Job " + id + " not in the awaited list");
        return;
    }
    logger.debug("Removing knowledge of job " + id);
    String pullUrl = aj.getPullURL();
    String pushUrl = aj.getPushURL();
    FileObject remotePullFolder = null;
    FileObject remotePushFolder = null;
    try {
        remotePullFolder = resolveFile(pullUrl);
        remotePushFolder = resolveFile(pushUrl);
    } catch (Exception e) {
        logger.error("Could not remove data for job " + id, e);
        return;
    }
    if (aj.isIsolateTaskOutputs()) {
        try {
            remotePullFolder = remotePullFolder.getParent();
        } catch (FileSystemException e) {
            logger.error("Could not get the parent of folder " + remotePullFolder, e);
        }
    }
    Set<FileObject> foldersToDelete = new HashSet<>();
    try {
        foldersToDelete.add(remotePullFolder.getParent());
        if (!remotePullFolder.getParent().equals(remotePushFolder.getParent()))
            foldersToDelete.add(remotePushFolder.getParent());
    } catch (FileSystemException e) {
        logger.warn("Data in folders " + pullUrl + " and " + pushUrl + " cannot be deleted due to an unexpected error ", e);
    }
    String url = "NOT YET DEFINED";
    for (FileObject fo : foldersToDelete) {
        try {
            url = fo.getURL().toString();
            if (!logger.isTraceEnabled()) {
                logger.debug("Deleting directory " + url);
                fo.delete(Selectors.SELECT_ALL);
                fo.delete();
            }
        } catch (FileSystemException e) {
            logger.warn("Could not delete temporary files at location " + url + " .", e);
        }
    }
    jobDatabase.removeAwaitedJob(id);
    try {
        jobDatabase.commit();
    } catch (IOException e) {
        logger.error("Could not save status file after removing job " + id, e);
    }
}

Example 40 with FileObject

use of org.apache.commons.vfs2.FileObject in project scheduling by ow2-proactive.

the class TestUserSpace method testUserSpace.

@Test
public void testUserSpace() throws Throwable {
    File in = tmpFolder.newFolder("input_space");
    String inPath = in.getAbsolutePath();
    File out = tmpFolder.newFolder("output_space");
    String outPath = out.getAbsolutePath();
    FileSystemManager fsManager = VFSFactory.createDefaultFileSystemManager();
    Scheduler sched = schedulerHelper.getSchedulerInterface();
    String userURI = sched.getUserSpaceURIs().get(0);
    assertTrue(userURI.startsWith("file:"));
    log("User URI is " + userURI);
    String userPath = new File(new URI(userURI)).getAbsolutePath();
    FileObject pathReplaceFO = fsManager.resolveFile(userURI + "/" + pathReplaceFile);
    if (pathReplaceFO.exists()) {
        pathReplaceFO.delete();
    }
    /**
     * Writes inFiles in INPUT
     */
    writeFiles(inFiles, inPath);
    File testPathRepl = new File(inPath + File.separator + pathReplaceFile);
    testPathRepl.createNewFile();
    PrintWriter out2 = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(testPathRepl))));
    out2.print(pathReplaceFile);
    out2.close();
    TaskFlowJob job = new TaskFlowJob();
    job.setName(this.getClass().getSimpleName());
    job.setInputSpace(in.toURI().toURL().toString());
    job.setOutputSpace(out.toURI().toURL().toString());
    JavaTask A = new JavaTask();
    A.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    A.setForkEnvironment(new ForkEnvironment());
    A.setName("A");
    for (String[] file : inFiles) {
        A.addInputFiles(file[0], InputAccessMode.TransferFromInputSpace);
        A.addOutputFiles(file[0] + ".glob.A", OutputAccessMode.TransferToUserSpace);
    }
    A.setPreScript(new SimpleScript(scriptA, "groovy"));
    job.addTask(A);
    JavaTask B = new JavaTask();
    B.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    B.setForkEnvironment(new ForkEnvironment());
    B.setName("B");
    B.addDependence(A);
    for (String[] file : inFiles) {
        B.addInputFiles(file[0] + ".glob.A", InputAccessMode.TransferFromUserSpace);
        B.addOutputFiles(file[0] + ".out", OutputAccessMode.TransferToOutputSpace);
    }
    B.setPreScript(new SimpleScript(scriptB, "groovy"));
    job.addTask(B);
    JobId id = sched.submit(job);
    schedulerHelper.waitForEventJobFinished(id);
    JobResult jr = schedulerHelper.getJobResult(id);
    Assert.assertFalse(jr.hadException());
    /**
     * check: inFiles > IN > LOCAL A > GLOBAL > LOCAL B > OUT
     */
    for (String[] inFile : inFiles) {
        File f = new File(outPath + File.separator + inFile[0] + ".out");
        assertTrue("File does not exist: " + f.getAbsolutePath(), f.exists());
        Assert.assertEquals("Original and copied files differ", inFile[1], FileUtils.readFileToString(f));
        File inf = new File(inPath + File.separator + inFile[0]);
    }
    for (String[] file : inFiles) {
        FileObject outFile = fsManager.resolveFile(userURI + "/" + file[0] + ".glob.A");
        log("Checking existence of " + outFile.getURL());
        assertTrue(outFile.getURL() + " exists", outFile.exists());
        File outFile2 = new File(userPath, file[0] + ".glob.A");
        log("Checking existence of " + outFile2);
        assertTrue(outFile2 + " exists", outFile2.exists());
    }
}