Examples with TaskFlowJob org.ow2.proactive.scheduler.common.job.TaskFlowJob used on opensource projects

Example 96 with TaskFlowJob

use of org.ow2.proactive.scheduler.common.job.TaskFlowJob in project scheduling by ow2-proactive.

the class TestNativeTaskPaths method testNativeTaskPaths.

@Test
public void testNativeTaskPaths() throws Throwable {
    File in = File.createTempFile("input", "space");
    in.delete();
    in.mkdir();
    String inPath = in.getAbsolutePath();
    File out = File.createTempFile("output", "space");
    out.delete();
    out.mkdir();
    File outc = new File(out, OutVarsFileC);
    File outd = new File(out, OutVarsFileD);
    if (outc.exists()) {
        outc.delete();
    }
    if (outd.exists()) {
        outd.delete();
    }
    File scriptTestEnv = null;
    if (OperatingSystem.getOperatingSystem() == OperatingSystem.unix) {
        scriptTestEnv = new File(inPath + File.separator + scriptCLinux);
        scriptTestEnv.createNewFile();
        PrintWriter out3 = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(scriptTestEnv))));
        out3.print(scriptCLinuxContent);
        out3.close();
    } else {
        scriptTestEnv = new File(inPath + File.separator + scriptCWindows);
        scriptTestEnv.createNewFile();
        PrintWriter out3 = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(scriptTestEnv))));
        out3.print(scriptCWindowsContent);
        out3.close();
    }
    TaskFlowJob job = new TaskFlowJob();
    job.setName(this.getClass().getSimpleName());
    job.setInputSpace(in.toURI().toURL().toString());
    job.setOutputSpace(out.toURI().toURL().toString());
    // // testing paths pattern
    // NativeTask C = new NativeTask();
    // C.setName("C");
    // C.addOutputFiles(OutVarsFileC, OutputAccessMode.TransferToOutputSpace);
    // switch (OperatingSystem.getOperatingSystem()) {
    // case windows:
    // C.setCommandLine(new String[] { "cmd", "/C",
    // "echo \"$JAVA\" \"$PROACTIVE_HOME\" > $LOCALSPACE\\" + OutVarsFileC });
    // break;
    // case unix:
    // C.setCommandLine(new String[] { "/bin/bash", "-c",
    // "echo \\\"$JAVA\\\" \\\"$PROACTIVE_HOME\\\" > $LOCALSPACE/" + OutVarsFileC });
    // break;
    // default:
    // throw new IllegalStateException("Unsupported operating system");
    // }
    // job.addTask(C);
    // testing $USERSPACE environment variable
    NativeTask D = new NativeTask();
    D.setName("D");
    if (OperatingSystem.getOperatingSystem() == OperatingSystem.unix) {
        D.addInputFiles(scriptCLinux, InputAccessMode.TransferFromInputSpace);
    } else {
        D.addInputFiles(scriptCWindows, InputAccessMode.TransferFromInputSpace);
    }
    D.addOutputFiles(OutVarsFileD, OutputAccessMode.TransferToOutputSpace);
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            D.setCommandLine(new String[] { "cmd", "/C", scriptCWindows });
            break;
        case unix:
            D.setCommandLine(new String[] { "/bin/bash", "-c", "chmod u+x $localspace/" + scriptCLinux + "; $localspace/" + scriptCLinux });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    D.setForkEnvironment(new ForkEnvironment("$LOCALSPACE"));
    job.addTask(D);
    Scheduler sched = schedulerHelper.getSchedulerInterface();
    JobId id = sched.submit(job);
    schedulerHelper.waitForEventJobFinished(id);
    String contentExpected = "foo";
    JobResult jr = schedulerHelper.getJobResult(id);
    Assert.assertFalse(jr.hadException());
    logger.info("Expected : '" + contentExpected + "'");
    // logger.info(jr.getAllResults().get("C").getOutput().getAllLogs(true));
    // String receivedc = IOUtils.toString(outc.toURI()).trim();
    // logger.info("Received C : '" + receivedc + "'");
    // Assert.assertEquals(contentExpected.toLowerCase(), receivedc.toLowerCase());
    logger.info(jr.getAllResults().get("D").getOutput().getAllLogs(true));
    String receivedd = IOUtils.toString(outd.toURI()).trim();
    logger.info("Received D : '" + receivedd + "'");
    Assert.assertEquals(contentExpected.toLowerCase(), receivedd.toLowerCase());
}

Example 97 with TaskFlowJob

use of org.ow2.proactive.scheduler.common.job.TaskFlowJob in project scheduling by ow2-proactive.

the class TestPropagatedVariables method createTaskFlowJob.

private TaskFlowJob createTaskFlowJob() throws UserException {
    TaskFlowJob flowJob = new TaskFlowJob();
    JavaTask taskA = new JavaTask();
    taskA.setName("Task_A");
    HashMap<String, String> setA = new HashMap<>();
    setA.put("Task_A_Var", "Task_A_Val");
    taskA.addArgument("set", setA);
    taskA.setExecutableClassName(PropagateVariablesExec.class.getName());
    flowJob.addTask(taskA);
    JavaTask taskB = new JavaTask();
    taskB.setName("Task_B");
    HashMap<String, String> setB = new HashMap<>();
    setB.put("Task_B_Var", "Task_B_Val");
    taskB.addArgument("set", setB);
    taskB.setExecutableClassName(PropagateVariablesExec.class.getName());
    flowJob.addTask(taskB);
    JavaTask taskC = new JavaTask();
    taskC.setName("Task_C");
    taskC.addDependence(taskA);
    taskC.addDependence(taskB);
    HashMap<String, String> checkC = new HashMap<>();
    checkC.put("Task_A_Var", "Task_A_Val");
    checkC.put("Task_B_Var", "Task_B_Val");
    taskC.addArgument("check", checkC);
    taskC.setExecutableClassName(PropagateVariablesExec.class.getName());
    flowJob.addTask(taskC);
    if (OperatingSystem.unix == OperatingSystem.getOperatingSystem()) {
        NativeTask taskD = new NativeTask();
        taskD.setName("TaskD");
        taskD.setCommandLine("/bin/bash", "-c", "echo $variables_Task_A_Var; test \"$variables_Task_A_Var\" == \"Task_A_Val\"");
        taskD.addDependence(taskC);
        flowJob.addTask(taskD);
    }
    return flowJob;
}

Example 98 with TaskFlowJob

use of org.ow2.proactive.scheduler.common.job.TaskFlowJob in project scheduling by ow2-proactive.

the class TestVariablesPatternLateBindings method testPropagatedVariableResolution.

@Test
public void testPropagatedVariableResolution() throws Throwable {
    TaskFlowJob job = (TaskFlowJob) StaxJobFactory.getFactory().createJob(absolutePath(job_desc));
    JobId id = schedulerHelper.submitJob(job);
    TaskInfo taskInfo = schedulerHelper.waitForEventTaskFinished(id, "Groovy_Task2");
    // assert the task finished correctly
    assertEquals(TaskStatus.FINISHED, taskInfo.getStatus());
    // for the second task:
    String jobLog = schedulerHelper.getTaskResult(id, "Groovy_Task2").getOutput().getAllLogs();
    Assert.assertThat(jobLog, CoreMatchers.containsString("WORKFLOW_VAR1=workflow_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("WORKFLOW_VAR2=var2_workflow_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("WORKFLOW_VAR3=workflow_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("WORKFLOW_VAR4=workflow_value_task_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("TASK_VAR1=task_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("TASK_VAR2=task_value_" + id.toString()));
    Assert.assertThat(jobLog, CoreMatchers.containsString("TASK_VAR3=task_value_task_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("TASK_VAR4=task_value_inherited_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("INHERITED_VAR1=inherited_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("INHERITED_VAR2=inherited_value_" + id.toString()));
    Assert.assertThat(jobLog, CoreMatchers.containsString("INHERITED_VAR3=inherited_value_task_value"));
    Assert.assertThat(jobLog, CoreMatchers.containsString("INHERITED_VAR4=inherited_value_inherited_value"));
}

Example 99 with TaskFlowJob

use of org.ow2.proactive.scheduler.common.job.TaskFlowJob in project scheduling by ow2-proactive.

the class JobEmailNotificationTest method createJob.

private InternalJob createJob(String userEmail) throws Exception {
    TaskFlowJob job = new TaskFlowJob();
    job.setName(JOB_NAME);
    if (userEmail != null) {
        job.addGenericInformation(JobEmailNotification.GENERIC_INFORMATION_KEY_EMAIL, userEmail);
    }
    JavaTask javaTask = new JavaTask();
    javaTask.setExecutableClassName(TestJavaTask.class.getName());
    javaTask.setName(TASK_NAME);
    job.addTask(javaTask);
    InternalJob internalJob = InternalJobFactory.createJob(job, null);
    internalJob.setOwner(DEFAULT_USER_NAME);
    return internalJob;
}

Example 100 with TaskFlowJob

use of org.ow2.proactive.scheduler.common.job.TaskFlowJob in project scheduling by ow2-proactive.

the class SchedulerJMXTest method action.

@Test
public void action() throws Exception {
    final String userLogin = TestUsers.DEMO.username;
    final String userPassword = TestUsers.DEMO.password;
    final String adminLogin = TestUsers.TEST.username;
    final String adminPassword = TestUsers.TEST.password;
    final SchedulerAuthenticationInterface auth = schedulerHelper.getSchedulerAuth();
    final PublicKey pubKey = auth.getPublicKey();
    // final Credentials userCreds =
    // Credentials.createCredentials(userLogin, userPassword, pubKey);
    final Credentials adminCreds = Credentials.createCredentials(new CredData(adminLogin, adminPassword), pubKey);
    final JMXServiceURL jmxRmiServiceURL = new JMXServiceURL(auth.getJMXConnectorURL(JMXTransportProtocol.RMI));
    final JMXServiceURL jmxRoServiceURL = new JMXServiceURL(auth.getJMXConnectorURL(JMXTransportProtocol.RO));
    final ObjectName allAccountsMBeanName = new ObjectName(SchedulerJMXHelper.ALLACCOUNTS_MBEAN_NAME);
    final ObjectName myAccountMBeanName = new ObjectName(SchedulerJMXHelper.MYACCOUNT_MBEAN_NAME);
    final ObjectName runtimeDataMBeanName = new ObjectName(SchedulerJMXHelper.RUNTIMEDATA_MBEAN_NAME);
    final ObjectName managementMBeanName = new ObjectName(SchedulerJMXHelper.MANAGEMENT_MBEAN_NAME);
    final String suffix = "/" + PASchedulerProperties.SCHEDULER_JMX_CONNECTOR_NAME.getValueAsString();
    {
        RMTHelper.log("Test jmxRmiServiceURL is well formed");
        assertTrue("The jmxRmiServiceURL protocol must be rmi", jmxRmiServiceURL.getProtocol().equals("rmi"));
        assertTrue("The jmxRmiServiceURL URLPath must end with " + suffix, jmxRmiServiceURL.getURLPath().endsWith(suffix));
    }
    {
        RMTHelper.log("Test jmxRoServiceURL is well formed");
        assertTrue("The jmxRoServiceURL protocol must be ro", jmxRoServiceURL.getProtocol().equals("ro"));
        assertTrue("The jmxRoServiceURL URLPath must end with " + suffix, jmxRoServiceURL.getURLPath().endsWith(suffix));
    }
    {
        log("Test jmxRmiServiceURL and jmxRoServiceURL are not equal");
        Assert.assertFalse("The jmxRmiServiceURL and jmxRoServiceURL must not be equal", jmxRmiServiceURL.equals(jmxRoServiceURL));
    }
    {
        log("Test invalid JMX auth without creds (expect SecurityException)");
        try {
            JMXConnectorFactory.connect(jmxRmiServiceURL, new HashMap<String, Object>(0));
        } catch (Exception e) {
            assertTrue("JMX auth must throw SecurityException if a client tries to connect without creds in the " + "env", e instanceof SecurityException);
        }
    }
    {
        log("Test invalid JMX auth with null login/password creds (expect SecurityException)");
        // Create the environment
        final HashMap<String, Object> env = new HashMap<String, Object>(1);
        env.put(JMXConnector.CREDENTIALS, new Object[] { null, null });
        try {
            JMXConnectorFactory.connect(jmxRmiServiceURL, env);
        } catch (Exception e) {
            assertTrue("JMX auth must throw SecurityException if a client tries to connect with null credentials" + " the env", e instanceof SecurityException);
        }
    }
    {
        log("Test invalid JMX auth with bad login/password creds");
        // Create the environment
        final HashMap<String, Object> env = new HashMap<>(1);
        env.put(JMXConnector.CREDENTIALS, new Object[] { "abra", "cadabra" });
        try {
            JMXConnectorFactory.connect(jmxRmiServiceURL, env);
        } catch (Exception e) {
            assertTrue("JMX auth must throw SecurityException if a client tries to connect with bad " + "login/password credentials the env", e instanceof SecurityException);
        }
    }
    // Tests as user over RMI
    {
        log("Test as user 1 - Auth with login/pass over RMI and check connection");
        // Create the environment
        final HashMap<String, Object> env = new HashMap<>(1);
        env.put(JMXConnector.CREDENTIALS, new Object[] { userLogin, userPassword });
        // Connect to the JMX RMI Connector Server
        final JMXConnector jmxConnector = JMXConnectorFactory.connect(jmxRmiServiceURL, env);
        final MBeanServerConnection conn = jmxConnector.getMBeanServerConnection();
        // Check that the MBean Server connection is not null
        assertNotNull("Unable to obtain the MBean server connection over RMI", conn);
        log("Test as user 2 - Check all mbeans are registered in the server");
        assertTrue("AllAccountsMBean is not registered", conn.isRegistered(allAccountsMBeanName));
        assertTrue("MyAccountMBean is not registered", conn.isRegistered(myAccountMBeanName));
        assertTrue("RuntimeDataMBean is not registered", conn.isRegistered(runtimeDataMBeanName));
        assertTrue("ManagementMBean is not registered", conn.isRegistered(managementMBeanName));
        log("Test as user 3 - Check MyAccountMBean attributes do not throw exceptions");
        final MBeanInfo info = conn.getMBeanInfo(myAccountMBeanName);
        for (final MBeanAttributeInfo att : info.getAttributes()) {
            final String attName = att.getName();
            try {
                conn.getAttribute(myAccountMBeanName, attName);
            } catch (Exception e) {
                fail("The attribute " + attName + " of MyAccountMBean must not throw " + e);
            }
        }
        log("Test as user 4 - Check RuntimeDataMBeanName attributes are correct");
        final String[] attributesToCheck = new String[] { "Status", "TotalJobsCount", "FinishedJobsCount", "TotalTasksCount", "FinishedTasksCount" };
        // Get all attributes to test BEFORE JOB SUBMISSION
        AttributeList list = conn.getAttributes(runtimeDataMBeanName, attributesToCheck);
        // Status
        Attribute att = (Attribute) list.get(0);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", "Started", att.getValue());
        // TotalJobsCount
        att = (Attribute) list.get(1);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", 0, att.getValue());
        // FinishedJobsCount
        att = (Attribute) list.get(2);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", 0, att.getValue());
        // NumberOfTasksCount
        att = (Attribute) list.get(3);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", 0, att.getValue());
        // FinishedTasksCount
        att = (Attribute) list.get(4);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", 0, att.getValue());
        // Create a job then submit it to the scheduler
        final int taskPerJob = 2;
        final TaskFlowJob job = new TaskFlowJob();
        for (int i = 0; i < taskPerJob; i++) {
            JavaTask task = new JavaTask();
            task.setName("" + i);
            task.setExecutableClassName(WaitAndPrint.class.getName());
            task.addArgument("sleepTime", "1");
            job.addTask(task);
        }
        // log as admin since its creds are already available
        final JobId id = schedulerHelper.submitJob(job);
        schedulerHelper.waitForEventJobFinished(id);
        // Get all attributes to test AFTER JOB EXECUTION
        list = conn.getAttributes(runtimeDataMBeanName, attributesToCheck);
        // Check SchedulerStatus
        att = (Attribute) list.get(0);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", "Started", att.getValue());
        // Check TotalNumberOfJobs
        att = (Attribute) list.get(1);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", 1, att.getValue());
        // Check NumberOfFinishedJobs
        att = (Attribute) list.get(2);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", 1, att.getValue());
        // Check TotalNumberOfTasks
        att = (Attribute) list.get(3);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", taskPerJob, att.getValue());
        // Check NumberOfFinishedTasks
        att = (Attribute) list.get(4);
        Assert.assertEquals("Incorrect value of " + att.getName() + " attribute", taskPerJob, att.getValue());
        jmxConnector.close();
    }
    // Test as admin over RO
    {
        log("Test as admin 1, auth with login/creds over RO and check connection");
        // Create the environment
        final HashMap<String, Object> env = new HashMap<>(1);
        env.put(JMXConnector.CREDENTIALS, new Object[] { adminLogin, adminCreds });
        env.put(JMXConnectorFactory.PROTOCOL_PROVIDER_PACKAGES, JMXProviderUtils.RO_PROVIDER_PKGS);
        // Connect to the JMX RO Connector Server
        final JMXConnector jmxConnector = JMXConnectorFactory.connect(jmxRoServiceURL, env);
        final MBeanServerConnection conn = jmxConnector.getMBeanServerConnection();
        // Check that the MBean Server connection is not null
        assertNotNull("Unable to obtain the MBean server connection over RO", conn);
        log("Test as admin 2 - Check ManagementMBean is registered in the MBean server");
        assertTrue("ManagementMBean is not registered", conn.isRegistered(managementMBeanName));
        RMTHelper.log("Test as admin 3 - Check ManagementMBean attributes do not throw exception");
        final MBeanInfo mInfo = conn.getMBeanInfo(managementMBeanName);
        for (final MBeanAttributeInfo att : mInfo.getAttributes()) {
            final String attName = att.getName();
            try {
                conn.getAttribute(managementMBeanName, attName);
            } catch (Exception e) {
                fail("The attribute " + attName + " of ManagementMBean must not throw " + e);
            }
        }
        RMTHelper.log("Test as admin 4 - Check AllAccountsMBean Username attribute");
        final String username = "Username";
        try {
            conn.setAttribute(allAccountsMBeanName, new Attribute(username, adminLogin));
        } catch (Exception e) {
            fail("Setting Username attribute of the AllAccountsMBean must not throw " + e);
        }
        String res = "";
        try {
            res = (String) conn.getAttribute(allAccountsMBeanName, username);
        } catch (Exception e) {
            fail("The attribute " + username + " of AllAccountsMBean must not throw " + e);
        }
        assertTrue("The attribute " + username + " of returns incorrect value", res.equals(adminLogin));
        jmxConnector.close();
    }
    // Test simultaneous RMI and RO connections
    {
        log("Test simultaneous JMX-RMI and JMX-RO connections as admin");
        final HashMap<String, Object> env = new HashMap<>(1);
        env.put(JMXConnector.CREDENTIALS, new Object[] { adminLogin, adminCreds });
        // Connect to the JMX-RMI Connector Server
        final JMXConnector jmxRmiConnector = JMXConnectorFactory.connect(jmxRmiServiceURL, env);
        final MBeanServerConnection conRmi = jmxRmiConnector.getMBeanServerConnection();
        // Connect to the JMX-RO Connector Server
        env.put(JMXConnectorFactory.PROTOCOL_PROVIDER_PACKAGES, JMXProviderUtils.RO_PROVIDER_PKGS);
        final JMXConnector jmxRoConnector1 = JMXConnectorFactory.connect(jmxRoServiceURL, env);
        final MBeanServerConnection conRo = jmxRoConnector1.getMBeanServerConnection();
        Assert.assertFalse("In case of simultaneous RMI and RO JMX connections they must not be equal", conRmi.equals(conRo));
        Assert.assertFalse("In case of simultaneous RMI and RO JMX connections the connectors must not provide the same connection ids", jmxRmiConnector.getConnectionId().equals(jmxRoConnector1.getConnectionId()));
        log("Test JMX-RO connection unicity (two connections over RO must not have the same id)");
        final JMXConnector jmxRoConnector2 = JMXConnectorFactory.connect(jmxRoServiceURL, env);
        Assert.assertFalse("In case of multiple RO JMX connections the connectors must not provide the same connection ids", jmxRoConnector1.getConnectionId().equals(jmxRoConnector2.getConnectionId()));
        // Close all connectors
        jmxRoConnector2.close();
        jmxRoConnector1.close();
        jmxRmiConnector.close();
    }
    // Test Helper class
    {
        log("Test JMXClientHelper as admin over RMI with connect() method");
        final JMXClientHelper client = new JMXClientHelper(auth, new Object[] { adminLogin, adminCreds });
        // default is over
        final boolean isConnected1 = client.connect();
        // RMI
        assertTrue("Unable to connect, exception is " + client.getLastException(), isConnected1);
        assertTrue("Incorrect default behavior of connect() method it must use RMI protocol", client.getConnector().getConnectionId().startsWith("rmi"));
        client.disconnect();
        Assert.assertFalse("The helper disconnect() must set the helper as disconnected", client.isConnected());
        final boolean isConnected2 = client.connect(JMXTransportProtocol.RO);
        assertTrue("Unable to connect, exception is " + client.getLastException(), isConnected2);
        assertTrue("The helper connect(JMXTransportProtocol.RO) method must use RO protocol", client.getConnector().getConnectionId().startsWith("ro"));
        client.disconnect();
        Assert.assertFalse("The helper disconnect() must set the helper as disconnected", client.isConnected());
    }
}