Examples with RemoteSource org.ow2.proactive.scheduler.rest.ds.RemoteSource used on opensource projects

Example 1 with RemoteSource

use of org.ow2.proactive.scheduler.rest.ds.RemoteSource in project scheduling by ow2-proactive.

the class RestJobTrackerImpl method removeAwaitedJob.

@Override
public void removeAwaitedJob(String id) {
    AwaitedJob aj = jobDatabase.getAwaitedJob(id);
    if (aj == null) {
        logger.warn("Job " + id + " not in the awaited list");
        return;
    }
    logger.debug("Removing knowledge of job " + id);
    String pullUrl = aj.getPullURL();
    String pushUrl = aj.getPushURL();
    Path remotePullFolder = null;
    Path remotePushFolder = null;
    try {
        remotePullFolder = Paths.get(new URI(pullUrl));
        remotePushFolder = Paths.get(new URI(pushUrl));
    } catch (Exception e) {
        logger.error("Could not remove data for job " + id, e);
        return;
    }
    if (aj.isIsolateTaskOutputs()) {
        Path tmp = remotePullFolder.getParent();
        if (tmp != null) {
            remotePullFolder = tmp;
        }
    }
    Set<Path> foldersToDelete = new HashSet<>();
    foldersToDelete.add(remotePullFolder.getParent());
    if (!remotePullFolder.getParent().equals(remotePushFolder.getParent())) {
        foldersToDelete.add(remotePushFolder.getParent());
    }
    RemoteSource remoteSource = new RemoteSource(IDataSpaceClient.Dataspace.USER);
    remoteSource.setType(FileType.FOLDER);
    for (Path path : foldersToDelete) {
        String location = path.toUri().toString();
        try {
            if (!logger.isTraceEnabled()) {
                logger.debug("Deleting directory " + location);
                remoteSource.setPath(location);
                restDataSpaceClient.delete(remoteSource);
            }
        } catch (NotConnectedException | PermissionException e) {
            logger.warn("Could not delete temporary files at location " + location + " .", e);
        }
    }
    jobDatabase.removeAwaitedJob(id);
    try {
        jobDatabase.commit();
    } catch (IOException e) {
        logger.error("Could not save status file after removing job " + id, e);
    }
}

Example 2 with RemoteSource

use of org.ow2.proactive.scheduler.rest.ds.RemoteSource in project scheduling by ow2-proactive.

the class RestJobTrackerImpl method removeAwaitedTask.

@Override
public void removeAwaitedTask(String id, String taskName) {
    AwaitedJob awaitedJob = jobDatabase.getAwaitedJob(id);
    if (awaitedJob == null) {
        logger.warn("Job " + id + " not in the awaited list");
        return;
    }
    AwaitedTask awaitedTask = awaitedJob.getAwaitedTask(taskName);
    if (awaitedTask == null) {
        logger.warn("Task " + taskName + " from Job " + id + " not in the awaited list");
        return;
    }
    logger.debug("Removing knowledge of task " + taskName + " from job " + id);
    if (awaitedJob.isIsolateTaskOutputs() && awaitedTask.getTaskId() != null) {
        // If the output data as been isolated in a dedicated folder we can delete it.
        String pullUrl = awaitedJob.getPullURL();
        pullUrl = pullUrl.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + awaitedTask.getTaskId());
        try {
            RemoteSource remoteSource = new RemoteSource(IDataSpaceClient.Dataspace.USER, pullUrl + "/");
            remoteSource.setType(FileType.FOLDER);
            restDataSpaceClient.delete(remoteSource);
        } catch (Throwable t) {
            logger.warn("Could not remove data for task " + taskName + " of job " + id, t);
        }
    }
    awaitedJob.removeAwaitedTask(taskName);
    if (awaitedJob.getAwaitedTasks().isEmpty()) {
        removeAwaitedJob(id);
        return;
    } else {
        // this is done to ensure persistence of the operation
        jobDatabase.putAwaitedJob(id, awaitedJob);
    }
    try {
        jobDatabase.commit();
    } catch (IOException e) {
        logger.error("Could not save status file after removing task Task " + taskName + " from Job" + id, e);
    }
}

Example 3 with RemoteSource

use of org.ow2.proactive.scheduler.rest.ds.RemoteSource in project scheduling by ow2-proactive.

the class RestSmartProxyImpl method downloadTaskOutputFiles.

@Override
protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, String taskName, String localFolder) throws NotConnectedException, PermissionException {
    AwaitedTask atask = awaitedjob.getAwaitedTask(taskName);
    if (atask == null) {
        throw new IllegalArgumentException("The task " + taskName + " does not belong to job " + jobId + " or has already been removed");
    }
    if (atask.isTransferring()) {
        logger.warn("The task " + taskName + " of job " + jobId + " is already transferring its output");
        return;
    }
    String outputSpace = awaitedjob.getOutputSpaceURL();
    String sourceFile;
    try {
        String userSpace = getLocalUserSpace();
        if (!outputSpace.startsWith(userSpace)) {
            logger.warn("RestSmartProxy does not support data transfers outside USERSPACE.");
        }
        sourceFile = outputSpace.substring(userSpace.length() + 1);
    } catch (Throwable error) {
        throw Throwables.propagate(error);
    }
    if (awaitedjob.isIsolateTaskOutputs()) {
        sourceFile = sourceFile.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + atask.getTaskId());
    }
    List<OutputSelector> outputFileSelectors = atask.getOutputSelectors();
    List<String> includes = Lists.newArrayList();
    List<String> excludes = Lists.newArrayList();
    if (outputFileSelectors != null) {
        for (OutputSelector os : outputFileSelectors) {
            addfileSelection(os.getOutputFiles(), includes, excludes);
        }
    }
    jobTracker.setTaskTransferring(jobId, taskName, true);
    if (awaitedjob.isAutomaticTransfer()) {
        threadPool.submit(new DownloadHandler(jobId, taskName, sourceFile, includes, excludes, localFolder));
    } else {
        try {
            RemoteSource source = new RemoteSource(USER, sourceFile);
            source.setIncludes(includes);
            source.setExcludes(excludes);
            File localDir = new File(localFolder);
            LocalDestination dest = new LocalDestination(localDir);
            restDataSpaceClient.download(source, dest);
        } catch (NotConnectedException | PermissionException e) {
            logger.error(String.format("Cannot download files, jobId=%s, taskId=%s, source=%s, destination=%s", jobId, taskName, sourceFile, localFolder), e);
            throw e;
        } finally {
            jobTracker.setTaskTransferring(jobId, taskName, false);
        }
        // task is removed from the job tracker only if the transfer is successful
        jobTracker.removeAwaitedTask(jobId, taskName);
    }
}

Example 4 with RemoteSource

use of org.ow2.proactive.scheduler.rest.ds.RemoteSource in project scheduling by ow2-proactive.

the class DataTransferTest method testCreateFolder.

@Test
public void testCreateFolder() throws Exception {
    URI srcDirPath = URI.create(getScheduler().getUserSpaceURIs().get(0));
    String folderName = "testcreatefolder";
    RemoteSource source = new RemoteSource(USER, folderName);
    source.setType(FileType.FOLDER);
    IDataSpaceClient client = clientInstance();
    assertTrue(client.create(source));
    File expectedFile = new File(srcDirPath.getPath(), folderName);
    assertTrue(expectedFile.exists());
    assertTrue(expectedFile.isDirectory());
}

Example 5 with RemoteSource

use of org.ow2.proactive.scheduler.rest.ds.RemoteSource in project scheduling by ow2-proactive.

the class DataTransferTest method testDownloadArchiveFile.

private void testDownloadArchiveFile(String archiveFileName, URL archiveSource) throws Exception {
    String srcDirPath = URI.create(getScheduler().getUserSpaceURIs().get(0)).getPath();
    File srcFile = new File(srcDirPath, archiveFileName);
    if (srcFile.exists()) {
        assertTrue(srcFile.delete());
    }
    FileUtils.copyInputStreamToFile(archiveSource.openStream(), srcFile);
    File tmpFile = tmpDir.newFile(archiveFileName);
    if (tmpFile.exists()) {
        assertTrue(tmpFile.delete());
    }
    // use standard client
    IDataSpaceClient client = clientInstance();
    RemoteSource source = new RemoteSource(USER, archiveFileName);
    LocalDestination dest = new LocalDestination(tmpFile);
    assertTrue(client.download(source, dest));
    assertTrue(Files.equal(srcFile, tmpFile));
    // use RemoteSpace API
    FileUtils.deleteQuietly(tmpFile);
    File downloadedFile = client.getUserSpace().pullFile(archiveFileName, tmpFile);
    assertTrue(Files.equal(srcFile, downloadedFile));
}