Examples with FileSystemServerDeployer org.objectweb.proactive.extensions.vfsprovider.FileSystemServerDeployer used on opensource projects

Example 1 with FileSystemServerDeployer

use of org.objectweb.proactive.extensions.vfsprovider.FileSystemServerDeployer in project scheduling by ow2-proactive.

the class TestJobDataspaceSubmission method testJobDataspaceSubmission.

/**
 * Tests start here.
 *
 * @throws Throwable any exception that can be thrown during the test.
 */
@org.junit.Test
public void testJobDataspaceSubmission() throws Throwable {
    // create initial directories and files
    setup();
    TaskFlowJob job = new TaskFlowJob();
    FileSystemServerDeployer filesServerIn = new FileSystemServerDeployer(IN, IOSPACE + IN, true, true);
    String url = filesServerIn.getVFSRootURL();
    job.setInputSpace(url);
    FileSystemServerDeployer filesServerOut = new FileSystemServerDeployer(OUT, IOSPACE + OUT, true);
    url = filesServerOut.getVFSRootURL();
    job.setOutputSpace(url);
    FileSystemServerDeployer filesServerGlob = new FileSystemServerDeployer(GLOB, IOSPACE + GLOB, true);
    url = filesServerGlob.getVFSRootURL();
    job.setGlobalSpace(url);
    FileSystemServerDeployer filesServerUser = new FileSystemServerDeployer(USER, IOSPACE + USER, true);
    url = filesServerUser.getVFSRootURL();
    job.setUserSpace(url);
    job.setName(this.getClass().getSimpleName());
    NativeTask t1 = new NativeTask();
    t1.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t1.addOutputFiles(out1, OutputAccessMode.TransferToOutputSpace);
    t1.setName("native_java1");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t1.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " > " + out1 });
            break;
        case unix:
            t1.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " > " + out1 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t1);
    NativeTask t2 = new NativeTask();
    t2.addInputFiles(in2, InputAccessMode.TransferFromOutputSpace);
    t2.addOutputFiles(out2, OutputAccessMode.TransferToOutputSpace);
    t2.setName("native_java2");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t2.setCommandLine(new String[] { "cmd", "/C", "type " + in2 + " > " + out2 });
            break;
        case unix:
            t2.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in2 + " > " + out2 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t2);
    NativeTask t3 = new NativeTask();
    t3.addInputFiles(in3, InputAccessMode.TransferFromGlobalSpace);
    t3.addOutputFiles(out3, OutputAccessMode.TransferToGlobalSpace);
    t3.setName("native_java3");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t3.setCommandLine(new String[] { "cmd", "/C", "type " + in3 + " > " + out3 });
            break;
        case unix:
            t3.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in3 + " > " + out3 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t3);
    NativeTask t4 = new NativeTask();
    t4.addInputFiles(in4, InputAccessMode.TransferFromUserSpace);
    t4.addOutputFiles(out4, OutputAccessMode.TransferToUserSpace);
    t4.setName("native_java4");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t4.setCommandLine(new String[] { "cmd", "/C", "type " + in4 + " > " + out4 });
            break;
        case unix:
            t4.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in4 + " > " + out4 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t4);
    NativeTask t10 = new NativeTask();
    t10.addDependence(t2);
    t10.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t10.addInputFiles(new FileSelector("*b.txt"), InputAccessMode.TransferFromOutputSpace);
    t10.addOutputFiles("*aa.txt", OutputAccessMode.TransferToOutputSpace);
    t10.setName("native_java10");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t10.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out2 + " > " + out10 });
            break;
        case unix:
            t10.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out2 + " > " + out10 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t10);
    NativeTask t11 = new NativeTask();
    t11.addDependence(t3);
    t11.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t11.addInputFiles(new FileSelector("*c.txt"), InputAccessMode.TransferFromGlobalSpace);
    t11.addOutputFiles("*bb.txt", OutputAccessMode.TransferToGlobalSpace);
    t11.setName("native_java11");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t11.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out3 + " > " + out11 });
            break;
        case unix:
            t11.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out3 + " > " + out11 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t11);
    NativeTask t12 = new NativeTask();
    t12.addDependence(t4);
    t12.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t12.addInputFiles(new FileSelector("*d.txt"), InputAccessMode.TransferFromUserSpace);
    t12.addOutputFiles("*cc.txt", OutputAccessMode.TransferToUserSpace);
    t12.setName("native_java7");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t12.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out4 + " > " + out12 });
            break;
        case unix:
            t12.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out4 + " > " + out12 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t12);
    JobId id = schedulerHelper.testJobSubmission(job);
    // check results are 0
    JobResult res = schedulerHelper.getJobResult(id);
    Assert.assertFalse(schedulerHelper.getJobResult(id).hadException());
    for (Map.Entry<String, TaskResult> entry : res.getAllResults().entrySet()) {
        Assert.assertEquals(0, entry.getValue().value());
    }
    // remove job
    schedulerHelper.removeJob(id);
    schedulerHelper.waitForEventJobRemoved(id);
    // check files
    File fout = new File(outputDir.getAbsolutePath() + File.separator + out1);
    Assert.assertEquals(in1, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out2);
    Assert.assertEquals(in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out3);
    Assert.assertEquals(in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out4);
    Assert.assertEquals(in4, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out10);
    Assert.assertEquals(in1 + in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out11);
    Assert.assertEquals(in1 + in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out12);
    Assert.assertEquals(in1 + in4, getContent(fout));
    // fout = new File(outputDir.getAbsolutePath() + File.separator + out20);
    // Assert.assertEquals(in1 + in2, getContent(fout));
    // fout = new File(globalDir.getAbsolutePath() + File.separator + out21);
    // Assert.assertEquals(in1 + in3, getContent(fout));
    // fout = new File(userDir.getAbsolutePath() + File.separator + out22);
    // Assert.assertEquals(in1 + in4, getContent(fout));
    filesServerIn.terminate();
    filesServerOut.terminate();
    filesServerGlob.terminate();
    filesServerUser.terminate();
    // and clean tmp space
    clean();
}

Example 2 with FileSystemServerDeployer

use of org.objectweb.proactive.extensions.vfsprovider.FileSystemServerDeployer in project scheduling by ow2-proactive.

the class TestSubmitJobWithPartiallyUnaccessibleDataSpaces method before.

@BeforeClass
public static void before() throws Throwable {
    spaceRoot = tmpFolder.newFolder("space");
    spaceRootUser = new File(spaceRoot, "demo");
    spaceRootUser.mkdirs();
    deployer = new FileSystemServerDeployer(spaceRoot.getAbsolutePath(), false);
    SchedulerTHelper.log("Dataspace started in : " + spaceRoot.getAbsolutePath());
    File inputFile = new File(spaceRootUser, "myfilein1");
    inputFile.createNewFile();
    File propertiesfile = new File(configFile.toURI());
    String propContent = FileUtils.readFileToString(propertiesfile, Charset.defaultCharset().toString());
    String newContent = propContent.replace("$$TOREPLACE$$", deployer.getVFSRootURL());
    FileUtils.writeStringToFile(propertiesfile, newContent);
    schedulerHelper = new SchedulerTHelper(true, propertiesfile.getAbsolutePath());
}

Example 3 with FileSystemServerDeployer

use of org.objectweb.proactive.extensions.vfsprovider.FileSystemServerDeployer in project scheduling by ow2-proactive.

the class DataSpaceServiceStarter method startServer.

private FileSystemServerDeployer startServer(String spaceName, String readableName, String spaceDir) throws IOException {
    FileSystemServerDeployer server = new FileSystemServerDeployer(spaceName, spaceDir, true, true);
    String[] urls = server.getVFSRootURLs();
    logger.info("Started " + readableName + " server at " + Arrays.toString(urls));
    return server;
}

Example 4 with FileSystemServerDeployer

use of org.objectweb.proactive.extensions.vfsprovider.FileSystemServerDeployer in project scheduling by ow2-proactive.

the class DataSpaceServiceStarter method startNamingService.

/**
 * StartNaming service and default file system server if needed.
 *
 * @throws Exception
 */
public void startNamingService() throws Exception {
    schedulerNode = (NodeImpl) PAActiveObject.getNode();
    localhostname = java.net.InetAddress.getLocalHost().getHostName();
    namingServiceDeployer = new NamingServiceDeployer(true);
    namingServiceURL = namingServiceDeployer.getNamingServiceURL();
    namingService = NamingService.createNamingServiceStub(namingServiceURL);
    // configure node for Data Spaces
    final BaseScratchSpaceConfiguration scratchConf = new BaseScratchSpaceConfiguration((String) null, DEFAULT_LOCAL_SCRATCH);
    DataSpacesNodes.configureNode(schedulerNode, scratchConf);
    // set default INPUT/OUTPUT spaces if needed
    PASchedulerProperties[][] confs = { { PASchedulerProperties.DATASPACE_DEFAULTINPUT_URL, PASchedulerProperties.DATASPACE_DEFAULTINPUT_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTINPUT_HOSTNAME }, { PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_URL, PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_HOSTNAME }, { PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_URL, PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_HOSTNAME }, { PASchedulerProperties.DATASPACE_DEFAULTUSER_URL, PASchedulerProperties.DATASPACE_DEFAULTUSER_LOCALPATH, PASchedulerProperties.DATASPACE_DEFAULTUSER_HOSTNAME } };
    String[] spacesNames = { "DefaultInputSpace", "DefaultOutputSpace", "GlobalSpace", "UserSpaces" };
    String[] humanReadableNames = { "default INPUT space", "default OUTPUT space", "shared GLOBAL space", "USER spaces" };
    String[] default_paths = { DEFAULT_LOCAL_INPUT, DEFAULT_LOCAL_OUTPUT, DEFAULT_LOCAL_GLOBAL, DEFAULT_LOCAL_USER };
    for (int i = 0; i < confs.length; i++) {
        // variable used to precise exception
        String spaceDir = null;
        if (confs[i][0].isSet() && confs[i][0].getValueAsString().trim().isEmpty()) {
            logger.info("Unsetting property : " + confs[i][0].getKey());
            confs[i][0].unSet();
        }
        if (!confs[i][0].isSet()) {
            // if URL is set, if not we start a server ourselves
            try {
                logger.debug("Starting " + humanReadableNames[i] + " server...");
                if (!confs[i][1].isSet()) {
                    // check if the localpath is set, if not we use the default path
                    spaceDir = default_paths[i];
                    confs[i][1].updateProperty(spaceDir);
                } else {
                    // otherwise, we build a FileServer on the provided path
                    logger.debug("Using property-defined path at " + confs[i][1].getValueAsString());
                    spaceDir = confs[i][1].getValueAsString();
                }
                File dir = new File(spaceDir);
                if (!dir.exists()) {
                    dir.mkdirs();
                }
                FileSystemServerDeployer server = startServer(spacesNames[i], humanReadableNames[i], spaceDir);
                servers.add(server);
                confs[i][0].updateProperty(buildServerUrlList(server.getVFSRootURLs()));
                // use the hostname property if it is set, otherwise, use the local hostname
                if (!confs[i][2].isSet()) {
                    confs[i][2].updateProperty(localhostname);
                }
                logger.debug(humanReadableNames[i] + " server local path is " + spaceDir);
            } catch (IllegalArgumentException iae) {
                throw new IllegalArgumentException("Directory '" + spaceDir + "' cannot be accessed. Check if directory exists or if you have read/write rights.");
            }
        }
    }
    Set<SpaceInstanceInfo> predefinedSpaces = new HashSet<>();
    namingService.registerApplication(SchedulerConstants.SCHEDULER_DATASPACE_APPLICATION_ID, predefinedSpaces);
    serviceStarted = true;
    try {
        // register the Global space
        createSpace(SchedulerConstants.SCHEDULER_DATASPACE_APPLICATION_ID, SchedulerConstants.GLOBALSPACE_NAME, PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_URL.getValueAsString(), PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_LOCALPATH.getValueAsString(), localhostname, false, true);
    } catch (Exception e) {
        logger.error("", e);
    }
}

Example 5 with FileSystemServerDeployer

use of org.objectweb.proactive.extensions.vfsprovider.FileSystemServerDeployer in project scheduling by ow2-proactive.

the class TestDataSpaceConfiguration method runStarter.

public Boolean runStarter() throws Exception {
    File spFile = new File(IOSPACE);
    File spFileWithUserDir = new File(IOSPACE, username);
    spFile.mkdirs();
    spFileWithUserDir.mkdirs();
    spFileWithUserDir.deleteOnExit();
    spFile.deleteOnExit();
    filesServerIn = new FileSystemServerDeployer("space", IOSPACE, true, true);
    String[] spaceurls = filesServerIn.getVFSRootURLs();
    String[] userdirUrls = DataSpaceServiceStarter.urlsWithUserDir(spaceurls, username);
    ArrayList<String> expected = new ArrayList<>();
    expected.addAll(Arrays.asList(spaceurls));
    ArrayList<String> expectedWithUserDir = new ArrayList<>();
    expectedWithUserDir.addAll(Arrays.asList(userdirUrls));
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTINPUT_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTOUTPUT_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTGLOBAL_HOSTNAME.updateProperty(HOSTNAME);
    PASchedulerProperties.DATASPACE_DEFAULTUSER_URL.updateProperty(DataSpaceServiceStarter.urlsToDSConfigProperty(spaceurls));
    PASchedulerProperties.DATASPACE_DEFAULTUSER_LOCALPATH.updateProperty(IOSPACE);
    PASchedulerProperties.DATASPACE_DEFAULTUSER_HOSTNAME.updateProperty(HOSTNAME);
    DataSpaceServiceStarter dsServiceStarter = DataSpaceServiceStarter.getDataSpaceServiceStarter();
    dsServiceStarter.startNamingService();
    Set<SpaceInstanceInfo> predefinedSpaces = new HashSet<>();
    NamingService namingService = dsServiceStarter.getNamingService();
    TaskDataSpaceApplication jdsa = new TaskDataSpaceApplication(appid, dsServiceStarter.getNamingService());
    jdsa.startDataSpaceApplication(null, null, null, null, username, null, null);
    DataSpacesNodes.configureApplication(PAActiveObject.getNode(), appid, dsServiceStarter.getNamingServiceURL());
    DataSpacesFileObject INPUT = PADataSpaces.resolveDefaultInput();
    DataSpacesFileObject OUTPUT = PADataSpaces.resolveDefaultOutput();
    DataSpacesFileObject GLOBAL = PADataSpaces.resolveOutput(SchedulerConstants.GLOBALSPACE_NAME);
    DataSpacesFileObject USER = PADataSpaces.resolveOutput(SchedulerConstants.USERSPACE_NAME);
    Assert.assertEquals(expectedWithUserDir, INPUT.getAllRealURIs());
    Assert.assertEquals(expectedWithUserDir, OUTPUT.getAllRealURIs());
    Assert.assertEquals(expected, GLOBAL.getAllRealURIs());
    Assert.assertEquals(expectedWithUserDir, USER.getAllRealURIs());
    jdsa.terminateDataSpaceApplication();
    return true;
}