Examples with LocalDestination org.ow2.proactive.scheduler.rest.ds.LocalDestination used on opensource projects

Example 1 with LocalDestination

use of org.ow2.proactive.scheduler.rest.ds.LocalDestination in project scheduling by ow2-proactive.

the class RestSmartProxyImpl method downloadTaskOutputFiles.

@Override
protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, String taskName, String localFolder) throws NotConnectedException, PermissionException {
    AwaitedTask atask = awaitedjob.getAwaitedTask(taskName);
    if (atask == null) {
        throw new IllegalArgumentException("The task " + taskName + " does not belong to job " + jobId + " or has already been removed");
    }
    if (atask.isTransferring()) {
        logger.warn("The task " + taskName + " of job " + jobId + " is already transferring its output");
        return;
    }
    String sourceFile;
    try {
        List<String> userSpaces = getUserSpaceURIs();
        List<String> outputSpaces = Arrays.asList(Tools.dataSpaceConfigPropertyToUrls(awaitedjob.getOutputSpaceURL()));
        sourceFile = extractRelativePath(userSpaces, outputSpaces);
        if (sourceFile == null) {
            if (awaitedjob.isAutomaticTransfer()) {
                logger.error("Could not extract remote path using inputSpace=" + outputSpaces + " and userSpace=" + userSpaces);
            } else {
                throw new IllegalArgumentException("Could not extract remote path using outputSpace=" + outputSpaces + " and userSpace=" + userSpaces);
            }
        }
    } catch (Throwable error) {
        throw Throwables.propagate(error);
    }
    if (awaitedjob.isIsolateTaskOutputs()) {
        sourceFile = sourceFile.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + atask.getTaskId());
    }
    List<OutputSelector> outputFileSelectors = atask.getOutputSelectors();
    List<String> includes = Lists.newArrayList();
    List<String> excludes = Lists.newArrayList();
    if (outputFileSelectors != null) {
        for (OutputSelector os : outputFileSelectors) {
            addfileSelection(os.getOutputFiles(), includes, excludes);
        }
    }
    jobTracker.setTaskTransferring(jobId, taskName, true);
    if (awaitedjob.isAutomaticTransfer()) {
        threadPool.submit(new DownloadHandler(jobId, taskName, sourceFile, includes, excludes, localFolder));
    } else {
        try {
            RemoteSource source = new RemoteSource(USER, sourceFile);
            source.setIncludes(includes);
            source.setExcludes(excludes);
            File localDir = new File(localFolder);
            LocalDestination dest = new LocalDestination(localDir);
            restDataSpaceClient.download(source, dest);
        } catch (NotConnectedException | PermissionException e) {
            logger.error(String.format("Cannot download files, jobId=%s, taskId=%s, source=%s, destination=%s", jobId, taskName, sourceFile, localFolder), e);
            throw e;
        } finally {
            jobTracker.setTaskTransferring(jobId, taskName, false);
        }
        // task is removed from the job tracker only if the transfer is successful
        jobTracker.removeAwaitedTask(jobId, taskName);
    }
}

Example 2 with LocalDestination

use of org.ow2.proactive.scheduler.rest.ds.LocalDestination in project scheduling by ow2-proactive.

the class DataTransferTest method testDownloadFile.

@Test
public void testDownloadFile() throws Exception {
    String srcDirPath = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    File srcFile = new File(srcDirPath, TEMP_FILE_TMP_NAME);
    if (srcFile.exists()) {
        assertTrue(srcFile.delete());
    }
    Files.write(randomFileContents(), srcFile);
    File tmpFile = tmpDir.newFile(TEMP_FILE_TMP_NAME);
    if (tmpFile.exists()) {
        assertTrue(tmpFile.delete());
    }
    // use standard client
    IDataSpaceClient client = clientInstance();
    RemoteSource source = new RemoteSource(USER, TEMP_FILE_TMP_NAME);
    LocalDestination dest = new LocalDestination(tmpFile);
    assertTrue(client.download(source, dest));
    assertTrue(Files.equal(srcFile, tmpFile));
    // use RemoteSpace API
    FileUtils.deleteQuietly(tmpFile);
    File downloadedFile = client.getUserSpace().pullFile(TEMP_FILE_TMP_NAME, tmpFile);
    assertTrue(Files.equal(srcFile, downloadedFile));
}

Example 3 with LocalDestination

use of org.ow2.proactive.scheduler.rest.ds.LocalDestination in project scheduling by ow2-proactive.

the class DataTransferTest method testDownloadArchiveFile.

private void testDownloadArchiveFile(String archiveFileName, URL archiveSource) throws Exception {
    String srcDirPath = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    File srcFile = new File(srcDirPath, archiveFileName);
    if (srcFile.exists()) {
        assertTrue(srcFile.delete());
    }
    FileUtils.copyInputStreamToFile(archiveSource.openStream(), srcFile);
    File tmpFile = tmpDir.newFile(archiveFileName);
    if (tmpFile.exists()) {
        assertTrue(tmpFile.delete());
    }
    // use standard client
    IDataSpaceClient client = clientInstance();
    RemoteSource source = new RemoteSource(USER, archiveFileName);
    LocalDestination dest = new LocalDestination(tmpFile);
    assertTrue(client.download(source, dest));
    assertTrue(Files.equal(srcFile, tmpFile));
    // use RemoteSpace API
    FileUtils.deleteQuietly(tmpFile);
    File downloadedFile = client.getUserSpace().pullFile(archiveFileName, tmpFile);
    assertTrue(Files.equal(srcFile, downloadedFile));
}

Example 4 with LocalDestination

use of org.ow2.proactive.scheduler.rest.ds.LocalDestination in project scheduling by ow2-proactive.

the class DataTransferTest method testDownloadAllFilesInDirectory.

@Test
public void testDownloadAllFilesInDirectory() throws Exception {
    String srcDirPath = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    String testFolderName = "testDownloadAllFilesInDirectory";
    System.out.println(testFolderName);
    TestFilesToDownloadCreator testFilesToDownloadCreator = new TestFilesToDownloadCreator(srcDirPath, testFolderName).invoke();
    File srcTextFile = testFilesToDownloadCreator.getSrcTextFile();
    File srcTempFile = testFilesToDownloadCreator.getSrcTempFile();
    File destTempDir = tmpDir.newFolder(TEMP_DIR_NAME);
    // use standard client
    IDataSpaceClient client = clientInstance();
    RemoteSource source = new RemoteSource(USER, testFolderName);
    LocalDestination dest = new LocalDestination(destTempDir);
    assertTrue(client.download(source, dest));
    assertTrue(Files.equal(srcTextFile, new File(destTempDir, TEMP_FILE_TXT_NAME)));
    assertTrue(Files.equal(srcTempFile, new File(destTempDir, TEMP_FILE_TMP_PATH)));
    // use RemoteSpace API
    File destTempDir2 = tmpDir.newFolder(TEMP_DIR2_NAME);
    client.getUserSpace().pullFile(testFolderName, destTempDir2);
    assertTrue(Files.equal(srcTextFile, new File(destTempDir2, TEMP_FILE_TXT_NAME)));
    assertTrue(Files.equal(srcTempFile, new File(destTempDir2, TEMP_FILE_TMP_PATH)));
}

Example 5 with LocalDestination

use of org.ow2.proactive.scheduler.rest.ds.LocalDestination in project scheduling by ow2-proactive.

the class DataTransferTest method testDownloadSelectedFilesUsingGlobPattern.

@Test
public void testDownloadSelectedFilesUsingGlobPattern() throws Exception {
    String srcDirPath = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    String testFolderName = "testDownloadSelectedFilesUsingGlobPattern";
    System.out.println(testFolderName);
    TestFilesToDownloadCreator testFilesToDownloadCreator = new TestFilesToDownloadCreator(srcDirPath, testFolderName).invoke();
    File srcTextFile = testFilesToDownloadCreator.getSrcTextFile();
    File srcTempFile = testFilesToDownloadCreator.getSrcTempFile();
    File destTempDir = tmpDir.newFolder(TEMP_DIR_NAME);
    // use standard client
    IDataSpaceClient client = clientInstance();
    RemoteSource source = new RemoteSource(USER, testFolderName);
    source.setIncludes("*.txt");
    LocalDestination dest = new LocalDestination(destTempDir);
    assertTrue(client.download(source, dest));
    File[] listFiles = destTempDir.listFiles();
    assertEquals(1, listFiles.length);
    assertTrue(Files.equal(srcTextFile, listFiles[0]));
    // use RemoteSpace API
    File destTempDir2 = tmpDir.newFolder(TEMP_DIR2_NAME);
    client.getUserSpace().pullFiles(testFolderName, "*.txt", destTempDir2);
    listFiles = destTempDir2.listFiles();
    assertEquals(1, listFiles.length);
    assertTrue(Files.equal(srcTextFile, listFiles[0]));
}