Examples with LocalDirSource org.ow2.proactive.scheduler.rest.ds.LocalDirSource used on opensource projects

Example 1 with LocalDirSource

use of org.ow2.proactive.scheduler.rest.ds.LocalDirSource in project scheduling by ow2-proactive.

the class DataTransferTest method testUploadSelectedFilesUsingFilenames.

@Test
public void testUploadSelectedFilesUsingFilenames() throws Exception {
    TestFilesToUploadCreator testFiles = new TestFilesToUploadCreator().invoke();
    File tempTextFile = testFiles.getTempTextFile();
    File tempFile = testFiles.getTempFile();
    // use standard client
    IDataSpaceClient client = clientInstance();
    LocalDirSource source = new LocalDirSource(tmpDir.getRoot());
    source.setIncludes("**/" + TEMP_FILE_TMP_NAME);
    RemoteDestination dest = new RemoteDestination(USER, "testUploadSelectedFilesUsingFilenames");
    assertTrue(client.upload(source, dest));
    String destRootUri = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    File[] destRootFiles = new File(destRootUri, "testUploadSelectedFilesUsingFilenames").listFiles();
    assertEquals(1, destRootFiles.length);
    assertTrue(Files.equal(tempFile, destRootFiles[0]));
    // use RemoteSpace API
    FileUtils.deleteDirectory(new File(destRootUri, "testUploadSelectedFilesUsingFilenames"));
    client.getUserSpace().pushFiles(tmpDir.getRoot(), "**/" + TEMP_FILE_TMP_NAME, "testUploadSelectedFilesUsingFilenames");
    destRootFiles = new File(destRootUri, "testUploadSelectedFilesUsingFilenames").listFiles();
    assertEquals(1, destRootFiles.length);
    assertTrue(Files.equal(tempFile, destRootFiles[0]));
}

Example 2 with LocalDirSource

use of org.ow2.proactive.scheduler.rest.ds.LocalDirSource in project scheduling by ow2-proactive.

the class RestSmartProxyImpl method uploadInputfiles.

/**
 * @throws NotConnectedException
 * @throws PermissionException
 * @see AbstractSmartProxy#uploadInputfiles(TaskFlowJob, String)
 */
@Override
public boolean uploadInputfiles(TaskFlowJob job, String localInputFolderPath) throws NotConnectedException, PermissionException {
    String userSpace = getLocalUserSpace();
    String inputSpace = job.getInputSpace();
    if (!inputSpace.startsWith(userSpace)) {
        // NOTE: only works for USERSPACE urls
        logger.warn("RestSmartProxy does not support data transfers outside USERSPACE.");
        return false;
    }
    String remotePath = inputSpace.substring(userSpace.length() + (userSpace.endsWith("/") ? 0 : 1));
    String jname = job.getName();
    logger.debug("Pushing files for job " + jname + " from " + localInputFolderPath + " to " + remotePath);
    TaskFlowJob tfj = job;
    for (Task t : tfj.getTasks()) {
        logger.debug("Pushing files for task " + t.getName());
        List<String> includes = Lists.newArrayList();
        List<String> excludes = Lists.newArrayList();
        List<InputSelector> inputFilesList = t.getInputFilesList();
        if (inputFilesList != null) {
            for (InputSelector is : inputFilesList) {
                addfileSelection(is.getInputFiles(), includes, excludes);
            }
        }
        LocalDirSource source = new LocalDirSource(localInputFolderPath);
        source.setIncludes(includes);
        source.setExcludes(excludes);
        RemoteDestination dest = new RemoteDestination(USER, remotePath);
        restDataSpaceClient.upload(source, dest);
    }
    logger.debug("Finished push operation from " + localInputFolderPath + " to " + remotePath);
    return true;
}

Example 3 with LocalDirSource

use of org.ow2.proactive.scheduler.rest.ds.LocalDirSource in project scheduling by ow2-proactive.

the class DataTransferTest method testUploadAllFilesInDirectory.

@Test
public void testUploadAllFilesInDirectory() throws Exception {
    // entire folder
    TestFilesToUploadCreator testFiles = new TestFilesToUploadCreator().invoke();
    File tempTextFile = testFiles.getTempTextFile();
    File tempFile = testFiles.getTempFile();
    // use standard client
    IDataSpaceClient client = clientInstance();
    LocalDirSource source = new LocalDirSource(tmpDir.getRoot());
    RemoteDestination dest = new RemoteDestination(USER, "testUploadAllFilesInDirectory");
    assertTrue(client.upload(source, dest));
    String destRootUri = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    assertTrue(Files.equal(tempTextFile, new File(destRootUri, "testUploadAllFilesInDirectory/" + TEMP_FILE_TXT_NAME)));
    assertTrue(Files.equal(tempFile, new File(destRootUri, "testUploadAllFilesInDirectory/" + TEMP_FILE_TMP_PATH)));
    // use RemoteSpace API
    FileUtils.deleteDirectory(new File(destRootUri, "testUploadAllFilesInDirectory"));
    client.getUserSpace().pushFile(tmpDir.getRoot(), "testUploadAllFilesInDirectory");
    assertTrue(Files.equal(tempTextFile, new File(destRootUri, "testUploadAllFilesInDirectory/" + TEMP_FILE_TXT_NAME)));
    assertTrue(Files.equal(tempFile, new File(destRootUri, "testUploadAllFilesInDirectory/" + TEMP_FILE_TMP_PATH)));
}

Example 4 with LocalDirSource

use of org.ow2.proactive.scheduler.rest.ds.LocalDirSource in project scheduling by ow2-proactive.

the class DataTransferTest method testUploadSelectedFilesUsingGlobPattern.

@Test
public void testUploadSelectedFilesUsingGlobPattern() throws Exception {
    TestFilesToUploadCreator testFiles = new TestFilesToUploadCreator().invoke();
    File tempTextFile = testFiles.getTempTextFile();
    File tempFile = testFiles.getTempFile();
    // use standard client
    IDataSpaceClient client = clientInstance();
    LocalDirSource source = new LocalDirSource(tmpDir.getRoot());
    source.setIncludes("*.txt");
    RemoteDestination dest = new RemoteDestination(USER, "testUploadSelectedFilesUsingGlobPattern");
    assertTrue(client.upload(source, dest));
    String destRootUri = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    File[] destRootFiles = new File(destRootUri, "testUploadSelectedFilesUsingGlobPattern").listFiles();
    assertEquals(1, destRootFiles.length);
    assertTrue(Files.equal(tempTextFile, destRootFiles[0]));
    // use RemoteSpace API
    FileUtils.deleteDirectory(new File(destRootUri, "testUploadSelectedFilesUsingGlobPattern"));
    client.getUserSpace().pushFiles(tmpDir.getRoot(), "*.txt", "testUploadSelectedFilesUsingGlobPattern");
    destRootFiles = new File(destRootUri, "testUploadSelectedFilesUsingGlobPattern").listFiles();
    assertEquals(1, destRootFiles.length);
    assertTrue(Files.equal(tempTextFile, destRootFiles[0]));
}