Examples with DataSpacesFileObject org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject used on opensource projects

Example 1 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method copyInputDataToScratch.

@Override
public void copyInputDataToScratch(List<InputSelector> inputSelectors) throws FileSystemException, InterruptedException {
    if (inputSelectors == null) {
        logger.debug("Input selector is empty, no file to copy");
        return;
    }
    ArrayList<DataSpacesFileObject> inputSpaceFiles = new ArrayList<>();
    ArrayList<DataSpacesFileObject> outputSpaceFiles = new ArrayList<>();
    ArrayList<DataSpacesFileObject> globalSpaceFiles = new ArrayList<>();
    ArrayList<DataSpacesFileObject> userSpaceFiles = new ArrayList<>();
    ArrayList<DataSpacesFileObject> inputSpaceCacheFiles = new ArrayList<>();
    ArrayList<DataSpacesFileObject> outputSpaceCacheFiles = new ArrayList<>();
    ArrayList<DataSpacesFileObject> globalSpaceCacheFiles = new ArrayList<>();
    ArrayList<DataSpacesFileObject> userSpaceCacheFiles = new ArrayList<>();
    findFilesToCopyFromInput(inputSelectors, inputSpaceFiles, outputSpaceFiles, globalSpaceFiles, userSpaceFiles, inputSpaceCacheFiles, outputSpaceCacheFiles, globalSpaceCacheFiles, userSpaceCacheFiles);
    String inputSpaceUri = virtualResolve(INPUT);
    String outputSpaceUri = virtualResolve(OUTPUT);
    String globalSpaceUri = virtualResolve(GLOBAL);
    String userSpaceUri = virtualResolve(USER);
    boolean cacheTransferPresent = !inputSpaceCacheFiles.isEmpty() || !outputSpaceCacheFiles.isEmpty() || !globalSpaceCacheFiles.isEmpty() || !userSpaceCacheFiles.isEmpty();
    if (cacheTransferPresent && CACHE != null) {
        cacheTransferLock.lockInterruptibly();
        try {
            Map<String, DataSpacesFileObject> filesToCopyToCache = createFolderHierarchySequentially(CACHE, inputSpaceUri, inputSpaceCacheFiles, outputSpaceUri, outputSpaceCacheFiles, globalSpaceUri, globalSpaceCacheFiles, userSpaceUri, userSpaceCacheFiles);
            long startTime = System.currentTimeMillis();
            List<Future<Boolean>> transferFuturesCache = doCopyInputDataToSpace(CACHE, filesToCopyToCache);
            handleResultsWhileTransferringFile(transferFuturesCache, "CACHE", startTime);
        } finally {
            if (cacheTransferPresent) {
                cacheTransferLock.unlock();
            }
        }
    } else if (cacheTransferPresent) {
        logDataspacesStatus("CACHE dataspace is not available while file transfers to cache were required. Check the Node logs for errors.", DataspacesStatusLevel.ERROR);
    }
    Map<String, DataSpacesFileObject> filesToCopyToScratch = createFolderHierarchySequentially(SCRATCH, inputSpaceUri, inputSpaceFiles, outputSpaceUri, outputSpaceFiles, globalSpaceUri, globalSpaceFiles, userSpaceUri, userSpaceFiles);
    long startTime = System.currentTimeMillis();
    List<Future<Boolean>> transferFuturesScratch = doCopyInputDataToSpace(SCRATCH, filesToCopyToScratch);
    handleResultsWhileTransferringFile(transferFuturesScratch, "LOCAL", startTime);
}

Example 2 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method copyScratchDataToOutput.

@Override
public void copyScratchDataToOutput(List<OutputSelector> outputSelectors) throws FileSystemException {
    if (outputSelectors == null) {
        logger.debug("Output selector is empty, no file to copy");
        return;
    }
    SCRATCH.refresh();
    checkOutputSpacesConfigured(outputSelectors);
    ArrayList<DataSpacesFileObject> results = new ArrayList<>();
    FileSystemException toBeThrown = null;
    for (OutputSelector os : outputSelectors) {
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector selector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
        selector.setIncludes(os.getOutputFiles().getIncludes());
        selector.setExcludes(os.getOutputFiles().getExcludes());
        switch(os.getMode()) {
            case TransferToOutputSpace:
                if (OUTPUT != null) {
                    toBeThrown = copyScratchDataToOutput(OUTPUT, "OUTPUT", os, selector, results);
                }
                break;
            case TransferToGlobalSpace:
                if (GLOBAL != null) {
                    toBeThrown = copyScratchDataToOutput(GLOBAL, "GLOBAL", os, selector, results);
                }
                break;
            case TransferToUserSpace:
                if (USER != null) {
                    toBeThrown = copyScratchDataToOutput(USER, "USER", os, selector, results);
                }
                break;
            default:
        }
        results.clear();
    }
    if (toBeThrown != null) {
        throw toBeThrown;
    }
}

Example 3 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method handleOutput.

private void handleOutput(final DataSpacesFileObject dataspaceDestination, String spaceName, org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector selector, List<DataSpacesFileObject> results) throws FileSystemException {
    long startTime = System.currentTimeMillis();
    Utils.findFiles(SCRATCH, selector, results);
    if (logger.isDebugEnabled()) {
        if (results == null || results.size() == 0) {
            logger.debug("No file found to copy from LOCAL space to " + spaceName + " space");
        } else {
            logger.debug("Files that will be copied from LOCAL space to " + spaceName + " space :");
        }
    }
    String base = SCRATCH.getVirtualURI();
    Map<String, DataSpacesFileObject> filesToCopy = new HashMap<>(results.size());
    createFolderHierarchySequentially(dataspaceDestination, base, results, filesToCopy);
    ArrayList<Future<Boolean>> transferFutures = new ArrayList<>(results.size());
    for (Map.Entry<String, DataSpacesFileObject> entry : filesToCopy.entrySet()) {
        transferFutures.add(parallelFileCopy(entry.getValue(), dataspaceDestination, entry.getKey(), false));
    }
    handleResultsWhileTransferringFile(transferFutures, spaceName, startTime);
}

Example 4 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method initDataSpace.

private DataSpacesFileObject initDataSpace(Callable<DataSpacesFileObject> dataSpaceBuilder, String dataSpaceName, boolean input) throws Exception {
    try {
        DataSpacesFileObject result = dataSpaceBuilder.call();
        result = resolveToExisting(result, dataSpaceName, input);
        result = createTaskIdFolder(result, dataSpaceName);
        // before the transfer
        if (result != null) {
            result.refresh();
        }
        return result;
    } catch (FileSystemException fse) {
        String message = dataSpaceName + " space is disabled";
        logger.warn(message, fse);
        logDataspacesStatus(message, DataspacesStatusLevel.WARNING);
        logDataspacesStatus(getStackTraceAsString(fse), DataspacesStatusLevel.WARNING);
    }
    return null;
}

Example 5 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class ServerJobAndTaskLogs method removePreciousLogs.

private void removePreciousLogs(JobId jobId, String jobOwner, Credentials credentials) {
    if (spacesSupport == null) {
        logger.warn("DataSpaces not initialized, cannot remove precious logs for job " + jobId);
        return;
    }
    try {
        Validate.notBlank(jobOwner);
        DataSpacesFileObject userspace = spacesSupport.getUserSpace(jobOwner, credentials);
        Validate.notNull(userspace, "Cannot find user space for user " + jobOwner + ". User spaces are probably not configured properly.");
        deleteAllLogFiles(jobId, userspace);
        deleteLogsFolderIfEmpty(jobId, userspace);
    } catch (Exception e) {
        logger.warn("Error occurred when trying to remove precious logs for job " + jobId, e);
    }
}