Examples with DataSpacesFileObject org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject used on opensource projects

Example 6 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method findFilesToCopyFromInput.

private void findFilesToCopyFromInput(List<InputSelector> inputSelectors, ArrayList<DataSpacesFileObject> inResults, ArrayList<DataSpacesFileObject> outResults, ArrayList<DataSpacesFileObject> globResults, ArrayList<DataSpacesFileObject> userResults, ArrayList<DataSpacesFileObject> inResultsCache, ArrayList<DataSpacesFileObject> outResultsCache, ArrayList<DataSpacesFileObject> globResultsCache, ArrayList<DataSpacesFileObject> userResultsCache) throws FileSystemException, InterruptedException {
    long startTime = System.currentTimeMillis();
    ArrayList<Future<List<DataSpacesFileObject>>> inResultsFutures = new ArrayList<>();
    ArrayList<Future<List<DataSpacesFileObject>>> outResultsFutures = new ArrayList<>();
    ArrayList<Future<List<DataSpacesFileObject>>> globResultsFutures = new ArrayList<>();
    ArrayList<Future<List<DataSpacesFileObject>>> userResultsFutures = new ArrayList<>();
    ArrayList<Future<List<DataSpacesFileObject>>> inResultsCacheFutures = new ArrayList<>();
    ArrayList<Future<List<DataSpacesFileObject>>> outResultsCacheFutures = new ArrayList<>();
    ArrayList<Future<List<DataSpacesFileObject>>> globResultsCacheFutures = new ArrayList<>();
    ArrayList<Future<List<DataSpacesFileObject>>> userResultsCacheFutures = new ArrayList<>();
    for (InputSelector is : inputSelectors) {
        org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector selector = new org.objectweb.proactive.extensions.dataspaces.vfs.selector.FileSelector();
        selector.setIncludes(is.getInputFiles().getIncludes());
        selector.setExcludes(is.getInputFiles().getExcludes());
        switch(is.getMode()) {
            case TransferFromInputSpace:
                inResultsFutures.add(findFilesToCopyFromInput(INPUT, "INPUT", is, selector));
                break;
            case TransferFromOutputSpace:
                outResultsFutures.add(findFilesToCopyFromInput(OUTPUT, "OUTPUT", is, selector));
                break;
            case TransferFromGlobalSpace:
                globResultsFutures.add(findFilesToCopyFromInput(GLOBAL, "GLOBAL", is, selector));
                break;
            case TransferFromUserSpace:
                userResultsFutures.add(findFilesToCopyFromInput(USER, "USER", is, selector));
                break;
            case CacheFromInputSpace:
                inResultsCacheFutures.add(findFilesToCopyFromInput(INPUT, "INPUT", is, selector));
                break;
            case CacheFromOutputSpace:
                outResultsCacheFutures.add(findFilesToCopyFromInput(OUTPUT, "OUTPUT", is, selector));
                break;
            case CacheFromGlobalSpace:
                globResultsCacheFutures.add(findFilesToCopyFromInput(GLOBAL, "GLOBAL", is, selector));
                break;
            case CacheFromUserSpace:
                userResultsCacheFutures.add(findFilesToCopyFromInput(USER, "USER", is, selector));
            case none:
                // do nothing
                break;
        }
    }
    addFilesResultToList(inResultsFutures, inResults);
    addFilesResultToList(outResultsFutures, outResults);
    addFilesResultToList(globResultsFutures, globResults);
    addFilesResultToList(userResultsFutures, userResults);
    addFilesResultToList(inResultsCacheFutures, inResultsCache);
    addFilesResultToList(outResultsCacheFutures, outResultsCache);
    addFilesResultToList(globResultsCacheFutures, globResultsCache);
    addFilesResultToList(userResultsCacheFutures, userResultsCache);
    logger.info("Time needed to create list of files to copy: " + (System.currentTimeMillis() - startTime) + " ms");
}

Example 7 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method setFolderRightsForRunAsUserMode.

/**
 * Sets open file permissions for folder copied in RunAsMe mode.
 * The method will set as well recursively the permissions on the parents folder.
 */
private void setFolderRightsForRunAsUserMode(DataSpacesFileObject object) throws FileSystemException {
    if (runAsUser) {
        setRWPermission(object);
        if (linuxOS) {
            object.setExecutable(true, false);
        }
        DataSpacesFileObject parentObject = null;
        try {
            parentObject = object.getParent();
        } catch (Exception ignored) {
        // in case getParent throws an exception instead of null, we prefer to ignore it and not propagate the permissions further.
        }
        if (parentObject != null) {
            setFolderRightsForRunAsUserMode(parentObject);
        }
    }
}

Example 8 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method createTaskIdFolder.

private DataSpacesFileObject createTaskIdFolder(DataSpacesFileObject space, String spaceName) {
    if (space != null) {
        String realURI = space.getRealURI();
        // Look for the TASKID pattern at the end of the dataspace URI
        if (realURI.contains(SchedulerConstants.TASKID_DIR_DEFAULT_NAME)) {
            // resolve the taskid subfolder
            DataSpacesFileObject tidOutput;
            try {
                tidOutput = space.resolveFile(taskId.toString());
                // create this subfolder
                tidOutput.createFolder();
            } catch (FileSystemException e) {
                logger.info("Error when creating the TASKID folder in " + realURI, e);
                logger.info(spaceName + " space is disabled");
                return null;
            }
            // assign it to the space
            space = tidOutput;
            logger.info(SchedulerConstants.TASKID_DIR_DEFAULT_NAME + " pattern found, changed " + spaceName + " space to : " + space.getRealURI());
        }
    }
    return space;
}

Example 9 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method createFolderHierarchySequentially.

/*
     * Create the folder hierarchy and select the files to copy
     * from the specified list of FileObjects.
     */
protected void createFolderHierarchySequentially(DataSpacesFileObject destination, String spaceUri, List<DataSpacesFileObject> spaceFiles, Map<String, DataSpacesFileObject> filesToCopy) throws FileSystemException {
    boolean isDebugEnabled = logger.isDebugEnabled();
    boolean isFolderHierarchyCreationEnabled = isCreateFolderHierarchySequentiallyEnabled();
    long startTime = System.currentTimeMillis();
    for (DataSpacesFileObject fileObject : spaceFiles) {
        String relativePath = relativize(spaceUri, fileObject);
        if (isFolderHierarchyCreationEnabled) {
            try {
                DataSpacesFileObject target = destination.resolveFile(relativePath);
                createFolderHierarchy(isDebugEnabled, fileObject, target);
            } catch (FileSystemException e) {
                String message = "Could not create folder hierarchy for " + relativePath + " on " + destination.getRealURI();
                logger.warn(message);
                logDataspacesStatus(message, DataspacesStatusLevel.WARNING);
            }
        }
        DataSpacesFileObject oldFileObject = filesToCopy.put(relativePath, fileObject);
        if (oldFileObject != null) {
            String message = fileObject.getRealURI() + " will be copied instead of " + oldFileObject.getRealURI() + ".\n " + "Precedence order is output space, input space, user space, global space.";
            logger.warn(message);
            logDataspacesStatus(message, DataspacesStatusLevel.WARNING);
        }
    }
    logger.info("Time needed to build folder hierarchy: " + (System.currentTimeMillis() - startTime) + " ms");
}

Example 10 with DataSpacesFileObject

use of org.objectweb.proactive.extensions.dataspaces.api.DataSpacesFileObject in project scheduling by ow2-proactive.

the class TestDataSpaceConfiguration method runStarter.

public Boolean runStarter() throws Exception {
    File spFile = new File(IOSPACE);
    File spFileWithUserDir = new File(IOSPACE, username);
    spFile.mkdirs();
    spFileWithUserDir.mkdirs();
    spFileWithUserDir.deleteOnExit();
    spFile.deleteOnExit();
    filesServerIn = new FileSystemServerDeployer("space", IOSPACE, true, true);
    String[] spaceurls = filesServerIn.getVFSRootURLs();
    String[] userdirUrls = DataSpaceServiceStarter.urlsWithUserDir(spaceurls, username);
    ArrayList<String> expected = new ArrayList<>();
    expected.addAll(Arrays.asList(spaceurls));
    ArrayList<String> expectedWithUserDir = new ArrayList<>();
    expectedWithUserDir.addAll(Arrays.asList(userdirUrls));
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTUSER_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTUSER_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTUSER_HOSTNAME.updateProperty(HOSTNAME);
    DataSpaceServiceStarter dsServiceStarter = DataSpaceServiceStarter.getDataSpaceServiceStarter();
    dsServiceStarter.startNamingService();
    Set<SpaceInstanceInfo> predefinedSpaces = new HashSet<>();
    NamingService namingService = dsServiceStarter.getNamingService();
    TaskDataSpaceApplication jdsa = new TaskDataSpaceApplication(appid, dsServiceStarter.getNamingService());
    jdsa.startDataSpaceApplication(null, null, null, null, username, null);
    DataSpacesNodes.configureApplication(PAActiveObject.getNode(), appid, dsServiceStarter.getNamingServiceURL());
    DataSpacesFileObject INPUT = PADataSpaces.resolveDefaultInput();
    DataSpacesFileObject OUTPUT = PADataSpaces.resolveDefaultOutput();
    DataSpacesFileObject GLOBAL = PADataSpaces.resolveOutput(SchedulerConstants.GLOBALSPACE_NAME);
    DataSpacesFileObject USER = PADataSpaces.resolveOutput(SchedulerConstants.USERSPACE_NAME);
    Assert.assertEquals(expectedWithUserDir, INPUT.getAllRealURIs());
    Assert.assertEquals(expectedWithUserDir, OUTPUT.getAllRealURIs());
    Assert.assertEquals(expected, GLOBAL.getAllRealURIs());
    Assert.assertEquals(expectedWithUserDir, USER.getAllRealURIs());
    jdsa.terminateDataSpaceApplication();
    return true;
}