Examples with SpaceInstanceInfo org.objectweb.proactive.extensions.dataspaces.core.SpaceInstanceInfo used on opensource projects

Example 1 with SpaceInstanceInfo

use of org.objectweb.proactive.extensions.dataspaces.core.SpaceInstanceInfo in project scheduling by ow2-proactive.

the class TaskProActiveDataspaces method initDataSpaces.

private void initDataSpaces() throws Exception {
    long startTime = System.currentTimeMillis();
    // configure node for application
    String appId = taskId.toString();
    // prepare scratch, input, output
    Node node = PAActiveObject.getNode();
    logger.info("Configuring dataspaces for app " + appId + " on " + node.getNodeInformation().getName());
    DataSpacesNodes.configureApplication(node, appId, namingService);
    SCRATCH = PADataSpaces.resolveScratchForAO();
    logger.info("SCRATCH space is " + SCRATCH.getRealURI());
    // Set the scratch folder writable for everyone
    if (!SCRATCH.setWritable(true, false)) {
        logger.warn("Missing permission to change write permissions to " + getScratchFolder());
    }
    InputOutputSpaceConfiguration cacheConfiguration = DataSpaceNodeConfigurationAgent.getCacheSpaceConfiguration();
    if (cacheConfiguration != null) {
        final String cacheName = cacheConfiguration.getName();
        cacheSpaceInstanceInfo = new SpaceInstanceInfo(appId, cacheConfiguration);
        try {
            namingService.register(cacheSpaceInstanceInfo);
        } catch (SpaceAlreadyRegisteredException e) {
            // this is a rare case where the cache space has already been registered for the same task and there was a node failure.
            namingService.unregister(cacheSpaceInstanceInfo.getMountingPoint());
            namingService.register(cacheSpaceInstanceInfo);
        }
        CACHE = initDataSpace(new Callable<DataSpacesFileObject>() {

            @Override
            public DataSpacesFileObject call() throws Exception {
                return PADataSpaces.resolveOutput(cacheName);
            }
        }, "CACHE", false);
    } else {
        logger.error("No Cache space configuration found, cache space is disabled.");
    }
    UserCredentials userCredentials;
    if (decrypter != null) {
        CredData credData = decrypter.decrypt();
        userCredentials = new UserCredentials(credData.getLogin(), credData.getPassword(), credData.getDomain(), credData.getKey());
    } else {
        logger.warn("No decryter found");
        userCredentials = new UserCredentials();
    }
    INPUT = initDataSpace(new Callable<DataSpacesFileObject>() {

        @Override
        public DataSpacesFileObject call() throws Exception {
            return PADataSpaces.resolveDefaultInput();
        }
    }, "INPUT", true);
    OUTPUT = initDataSpace(new Callable<DataSpacesFileObject>() {

        @Override
        public DataSpacesFileObject call() throws Exception {
            return PADataSpaces.resolveDefaultOutput();
        }
    }, "OUTPUT", false);
    GLOBAL = initDataSpace(new Callable<DataSpacesFileObject>() {

        @Override
        public DataSpacesFileObject call() throws Exception {
            return PADataSpaces.resolveOutput(SchedulerConstants.GLOBALSPACE_NAME);
        }
    }, "GLOBAL", false);
    USER = initDataSpace(new Callable<DataSpacesFileObject>() {

        @Override
        public DataSpacesFileObject call() throws Exception {
            return PADataSpaces.resolveOutput(SchedulerConstants.USERSPACE_NAME, userCredentials);
        }
    }, "USER", false);
    logger.info("Time needed to mount data spaces: " + (System.currentTimeMillis() - startTime) + " ms");
}

Example 2 with SpaceInstanceInfo

use of org.objectweb.proactive.extensions.dataspaces.core.SpaceInstanceInfo in project scheduling by ow2-proactive.

the class DataSpaceServiceStarter method startNamingService.

/**
 * StartNaming service and default file system server if needed.
 *
 * @throws Exception
 */
public void startNamingService() throws Exception {
    schedulerNode = (NodeImpl) PAActiveObject.getNode();
    localhostname = java.net.InetAddress.getLocalHost().getHostName();
    namingServiceDeployer = new NamingServiceDeployer(true);
    namingServiceURL = namingServiceDeployer.getNamingServiceURL();
    namingService = NamingService.createNamingServiceStub(namingServiceURL);
    // configure node for Data Spaces
    final BaseScratchSpaceConfiguration scratchConf = new BaseScratchSpaceConfiguration((String) null, DEFAULT_LOCAL_SCRATCH);
    DataSpacesNodes.configureNode(schedulerNode, scratchConf);
    // set default INPUT/OUTPUT spaces if needed
    PASchedulerProperties[][] confs = { { PASchedulerProperties.DATASPACE_DEFAULTINPUT_URL, PASchedulerProperties.DATASPACE_DEFAULTINPUT_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTINPUT_HOSTNAME }, { PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_URL, PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_HOSTNAME }, { PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_URL, PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_HOSTNAME }, { PASchedulerProperties.DATASPACE_DEFAULTUSER_URL, PASchedulerProperties.DATASPACE_DEFAULTUSER_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTUSER_HOSTNAME } };
    String[] spacesNames = { "DefaultInputSpace", "DefaultOutputSpace", "GlobalSpace", "UserSpaces" };
    String[] humanReadableNames = { "default INPUT space", "default OUTPUT space", "shared GLOBAL space", "USER spaces" };
    String[] default_paths = { DEFAULT_LOCAL_INPUT, DEFAULT_LOCAL_OUTPUT, DEFAULT_LOCAL_GLOBAL, DEFAULT_LOCAL_USER };
    for (int i = 0; i < confs.length; i++) {
        // variable used to precise exception
        String spaceDir = null;
        if (confs[i][0].isSet() && confs[i][0].getValueAsString().trim().isEmpty()) {
            logger.info("Unsetting property : " + confs[i][0].getKey());
            confs[i][0].unSet();
        }
        if (!confs[i][0].isSet()) {
            // if URL is set, if not we start a server ourselves
            try {
                logger.debug("Starting " + humanReadableNames[i] + " server...");
                if (!confs[i][1].isSet()) {
                    // check if the localpath is set, if not we use the default path
                    spaceDir = default_paths[i];
                    confs[i][1].updateProperty(spaceDir);
                } else {
                    // otherwise, we build a FileServer on the provided path
                    logger.debug("Using property-defined path at " + confs[i][1].getValueAsString());
                    spaceDir = confs[i][1].getValueAsString();
                }
                File dir = new File(spaceDir);
                if (!dir.exists()) {
                    dir.mkdirs();
                }
                FileSystemServerDeployer server = startServer(spacesNames[i], humanReadableNames[i], spaceDir);
                servers.add(server);
                confs[i][0].updateProperty(buildServerUrlList(server.getVFSRootURLs()));
                // use the hostname property if it is set, otherwise, use the local hostname
                if (!confs[i][2].isSet()) {
                    confs[i][2].updateProperty(localhostname);
                }
                logger.debug(humanReadableNames[i] + " server local path is " + spaceDir);
            } catch (IllegalArgumentException iae) {
                throw new IllegalArgumentException("Directory '" + spaceDir + "' cannot be accessed. Check if directory exists or if you have read/write rights.");
            }
        }
    }
    Set<SpaceInstanceInfo> predefinedSpaces = new HashSet<>();
    namingService.registerApplication(SchedulerConstants.SCHEDULER_DATASPACE_APPLICATION_ID, predefinedSpaces);
    serviceStarted = true;
    try {
        // register the Global space
        createSpace(SchedulerConstants.SCHEDULER_DATASPACE_APPLICATION_ID, SchedulerConstants.GLOBALSPACE_NAME, PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_URL.getValueAsString(), PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_LOCALPATH.getValueAsString(), localhostname, false, true);
    } catch (Exception e) {
        logger.error("", e);
    }
}

Example 3 with SpaceInstanceInfo

use of org.objectweb.proactive.extensions.dataspaces.core.SpaceInstanceInfo in project scheduling by ow2-proactive.

the class DataSpaceServiceStarter method createSpace.

/**
 * Helper method used to create a space configuration and register it into the naming service
 * This helper can eventually configure the local Node for the provided application ID, if the dataspace needs to be user locally.
 * It will only register the dataspace in the naming service otherwise
 * @param appID the Application ID
 * @param name the name of the dataspace
 * @param urlsproperty the space-delimited url list property of the Virtual File Systems (for different protocols)
 * @param path the path to the dataspace in the localfilesystem
 * @param hostname the host where the file server is deployed
 * @param inputConfiguration if the configuration is an InputSpace configuration (read-only)
 * @param localConfiguration if the local node needs to be configured for the provided application
 */
public void createSpace(String appID, String name, String urlsproperty, String path, String hostname, boolean inputConfiguration, boolean localConfiguration) throws FileSystemException, URISyntaxException, ProActiveException, MalformedURLException {
    if (!serviceStarted) {
        throw new IllegalStateException("DataSpace service is not started");
    }
    if (!spacesConfigurations.containsKey(appID)) {
        if (localConfiguration) {
            if (appidConfigured != null) {
                logger.warn("Node " + schedulerNode.getNodeInformation().getURL() + " was configured for appid = " + appidConfigured + ", reconfiguring...");
            }
            DataSpacesNodes.configureApplication(schedulerNode, appID, namingService);
            logger.debug("Node " + schedulerNode.getNodeInformation().getURL() + " configured for appid = " + appID);
            appidConfigured = appID;
        } else {
            namingService.registerApplication(appID, new HashSet<SpaceInstanceInfo>());
        }
        spacesConfigurations.put(appID, new HashSet<String>());
    }
    if (spacesConfigurations.get(appID).contains(name) && !PADataSpaces.DEFAULT_IN_OUT_NAME.equals(name)) {
        throw new SpaceAlreadyRegisteredException("Space " + name + " for appid=" + appID + " is already registered");
    }
    InputOutputSpaceConfiguration spaceConf = null;
    // Converts the property to an ArrayList
    ArrayList<String> finalurls = new ArrayList<>(Arrays.asList(Tools.dataSpaceConfigPropertyToUrls(urlsproperty)));
    if (inputConfiguration) {
        spaceConf = InputOutputSpaceConfiguration.createInputSpaceConfiguration(finalurls, path, hostname != null ? hostname : localhostname, name);
    } else {
        spaceConf = InputOutputSpaceConfiguration.createOutputSpaceConfiguration(finalurls, path, hostname != null ? hostname : localhostname, name);
    }
    namingService.register(new SpaceInstanceInfo(appID, spaceConf));
    spacesConfigurations.get(appID).add(spaceConf.getName());
    logger.debug("Space " + name + " for appid = " + appID + " with urls = " + finalurls + " registered");
}

Example 4 with SpaceInstanceInfo

use of org.objectweb.proactive.extensions.dataspaces.core.SpaceInstanceInfo in project scheduling by ow2-proactive.

the class TestDataSpaceConfiguration method runStarter.

public Boolean runStarter() throws Exception {
    File spFile = new File(IOSPACE);
    File spFileWithUserDir = new File(IOSPACE, username);
    spFile.mkdirs();
    spFileWithUserDir.mkdirs();
    spFileWithUserDir.deleteOnExit();
    spFile.deleteOnExit();
    filesServerIn = new FileSystemServerDeployer("space", IOSPACE, true, true);
    String[] spaceurls = filesServerIn.getVFSRootURLs();
    String[] userdirUrls = DataSpaceServiceStarter.urlsWithUserDir(spaceurls, username);
    ArrayList<String> expected = new ArrayList<>();
    expected.addAll(Arrays.asList(spaceurls));
    ArrayList<String> expectedWithUserDir = new ArrayList<>();
    expectedWithUserDir.addAll(Arrays.asList(userdirUrls));
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTUSER_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTUSER_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTUSER_HOSTNAME.updateProperty(HOSTNAME);
    DataSpaceServiceStarter dsServiceStarter = DataSpaceServiceStarter.getDataSpaceServiceStarter();
    dsServiceStarter.startNamingService();
    Set<SpaceInstanceInfo> predefinedSpaces = new HashSet<>();
    NamingService namingService = dsServiceStarter.getNamingService();
    TaskDataSpaceApplication jdsa = new TaskDataSpaceApplication(appid, dsServiceStarter.getNamingService());
    jdsa.startDataSpaceApplication(null, null, null, null, username, null, null);
    DataSpacesNodes.configureApplication(PAActiveObject.getNode(), appid, dsServiceStarter.getNamingServiceURL());
    DataSpacesFileObject INPUT = PADataSpaces.resolveDefaultInput();
    DataSpacesFileObject OUTPUT = PADataSpaces.resolveDefaultOutput();
    DataSpacesFileObject GLOBAL = PADataSpaces.resolveOutput(SchedulerConstants.GLOBALSPACE_NAME);
    DataSpacesFileObject USER = PADataSpaces.resolveOutput(SchedulerConstants.USERSPACE_NAME);
    Assert.assertEquals(expectedWithUserDir, INPUT.getAllRealURIs());
    Assert.assertEquals(expectedWithUserDir, OUTPUT.getAllRealURIs());
    Assert.assertEquals(expected, GLOBAL.getAllRealURIs());
    Assert.assertEquals(expectedWithUserDir, USER.getAllRealURIs());
    jdsa.terminateDataSpaceApplication();
    return true;
}