Examples with SelectionScript org.ow2.proactive.scripting.SelectionScript used on opensource projects

Example 36 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class StaticSelectionScriptTest method action.

@Test
public void action() throws Exception {
    ResourceManager resourceManager = rmHelper.getResourceManager();
    int nodeNumber = rmHelper.createNodeSource("StaticSelectionScriptTest");
    String node1Name = "node1";
    String node2Name = "node2";
    HashMap<String, String> vmProperties = new HashMap<>();
    String vmPropKey = "myProperty";
    String vmPropValue = "myValue";
    vmProperties.put(vmPropKey, vmPropValue);
    TestNode node1 = rmHelper.createNode(node1Name, vmProperties);
    testNodes.add(node1);
    String node1URL = node1.getNode().getNodeInformation().getURL();
    resourceManager.addNode(node1URL);
    // wait node adding event
    rmHelper.waitForNodeEvent(RMEventType.NODE_ADDED, node1URL);
    // wait for the nodes to be in free state
    rmHelper.waitForAnyNodeEvent(RMEventType.NODE_STATE_CHANGED);
    // create the static selection script object
    SelectionScript sScript = new SelectionScript(new File(vmPropSelectionScriptpath.toURI()), new String[] { vmPropKey, vmPropValue }, false);
    log("Test 1");
    NodeSet nodes = resourceManager.getAtMostNodes(1, sScript);
    // wait node selection
    PAFuture.waitFor(nodes);
    assertEquals(1, nodes.size());
    assertEquals(nodeNumber, resourceManager.getState().getFreeNodesNumber());
    assertEquals(node1URL, nodes.get(0).getNodeInformation().getURL());
    // wait for node busy event
    RMNodeEvent evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node1URL);
    assertEquals(NodeState.BUSY, evt.getNodeState());
    resourceManager.releaseNode(nodes.get(0));
    // wait for node free event
    evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node1URL);
    assertEquals(NodeState.FREE, evt.getNodeState());
    log("Test 2");
    nodes = resourceManager.getAtMostNodes(3, sScript);
    // wait node selection
    PAFuture.waitFor(nodes);
    assertEquals(1, nodes.size());
    assertEquals(nodeNumber, resourceManager.getState().getFreeNodesNumber());
    assertEquals(node1URL, nodes.get(0).getNodeInformation().getURL());
    // wait for node busy event
    evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node1URL);
    assertEquals(NodeState.BUSY, evt.getNodeState());
    resourceManager.releaseNode(nodes.get(0));
    // wait for node free event
    evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node1URL);
    assertEquals(NodeState.FREE, evt.getNodeState());
    log("Test 3");
    // add a second with JVM env var
    TestNode node2 = rmHelper.createNode(node2Name, vmProperties);
    testNodes.add(node2);
    String node2URL = node2.getNode().getNodeInformation().getURL();
    resourceManager.addNode(node2URL);
    // wait node adding event
    rmHelper.waitForNodeEvent(RMEventType.NODE_ADDED, node2URL);
    // wait for the nodes to be in free state
    rmHelper.waitForAnyNodeEvent(RMEventType.NODE_STATE_CHANGED);
    nodes = resourceManager.getAtMostNodes(3, sScript);
    // wait node selection
    PAFuture.waitFor(nodes);
    assertEquals(2, nodes.size());
    assertEquals(nodeNumber, resourceManager.getState().getFreeNodesNumber());
    // wait for node busy event
    evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node1URL);
    assertEquals(NodeState.BUSY, evt.getNodeState());
    evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node2URL);
    assertEquals(NodeState.BUSY, evt.getNodeState());
    resourceManager.releaseNodes(nodes);
    // wait for nodes free event
    evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node1URL);
    assertEquals(NodeState.FREE, evt.getNodeState());
    evt = rmHelper.waitForNodeEvent(RMEventType.NODE_STATE_CHANGED, node2URL);
    Assert.assertEquals(NodeState.FREE, evt.getNodeState());
    log("Test 4");
    resourceManager.removeNode(node1URL, true);
    resourceManager.removeNode(node2URL, true);
    // wait for node removed event
    rmHelper.waitForNodeEvent(RMEventType.NODE_REMOVED, node1URL);
    rmHelper.waitForNodeEvent(RMEventType.NODE_REMOVED, node2URL);
    nodes = resourceManager.getAtMostNodes(3, sScript);
    // wait node selection
    PAFuture.waitFor(nodes);
    assertEquals(0, nodes.size());
    assertEquals(nodeNumber, resourceManager.getState().getFreeNodesNumber());
    log("Test 5");
    // create the bad static selection script object
    SelectionScript badScript = new SelectionScript(new File(badSelectionScriptpath.toURI()), new String[] {}, false);
    nodes = resourceManager.getAtMostNodes(3, badScript);
    // wait node selection
    PAFuture.waitFor(nodes);
    assertEquals(0, nodes.size());
    assertEquals(nodeNumber, resourceManager.getState().getFreeNodesNumber());
    log("Test 6");
    // create the static selection script object that doesn't define 'selected'
    SelectionScript noSelectedScript = new SelectionScript(new File(withoutSelectedSelectionScriptpath.toURI()), new String[] {}, false);
    nodes = resourceManager.getAtMostNodes(3, noSelectedScript);
    // wait node selection
    PAFuture.waitFor(nodes);
    assertEquals(0, nodes.size());
    assertEquals(nodeNumber, resourceManager.getState().getFreeNodesNumber());
}

Example 37 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class Task method displaySelectionScripts.

private String displaySelectionScripts() {
    if (sScripts == null)
        return null;
    String answer = "{ ";
    for (int i = 0; i < sScripts.size(); i++) {
        SelectionScript ss = sScripts.get(i);
        if (ss != null) {
            answer += ss.display() + ((i < sScripts.size() - 1) ? " , " : "");
        }
    }
    answer += " }";
    return answer;
}

Example 38 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class RMCore method getNodes.

/**
 * {@inheritDoc}
 */
public NodeSet getNodes(int number, TopologyDescriptor topology, List<SelectionScript> selectionScrips, NodeSet exclusion, boolean bestEffort) {
    Criteria criteria = new Criteria(number);
    criteria.setTopology(topology);
    criteria.setScripts(selectionScrips);
    criteria.setBlackList(exclusion);
    criteria.setBestEffort(bestEffort);
    return getNodes(criteria);
}

Example 39 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method getRMNodes.

/**
 * Ask to the RM the given number of node resources.<br>
 * If there is a problem with these task selection (such as bad selectionScript) this method
 * will terminate the corresponding tasks and jobs. As the selection scripts contain errors, the task
 * and its surrounding jobs must be stopped.
 *
 * @param neededResourcesNumber the number of resources to ask for (must be &gt; 0).
 * @param tasksToSchedule the task to be scheduled
 * @return A nodeSet that contains at most 'neededResourcesNumber' available compatible resources.
 * 		   An empty nodeSet if no nodes could be found
 * 		   null if the their was an exception when asking for the nodes (ie : selection script has failed)
 */
protected NodeSet getRMNodes(Map<JobId, JobDescriptor> jobMap, int neededResourcesNumber, LinkedList<EligibleTaskDescriptor> tasksToSchedule, Set<String> freeResources) {
    NodeSet nodeSet;
    if (neededResourcesNumber <= 0) {
        throw new IllegalArgumentException("'neededResourcesNumber' must be greater than 0");
    }
    EligibleTaskDescriptor etd = tasksToSchedule.getFirst();
    InternalJob currentJob = ((JobDescriptorImpl) jobMap.get(etd.getJobId())).getInternal();
    InternalTask internalTask0 = currentJob.getIHMTasks().get(etd.getTaskId());
    try {
        TopologyDescriptor descriptor = null;
        boolean bestEffort = true;
        if (internalTask0.isParallel()) {
            descriptor = internalTask0.getParallelEnvironment().getTopologyDescriptor();
            bestEffort = false;
            if (descriptor == null) {
                logger.debug("Topology is not defined for the task " + internalTask0.getName());
            }
        }
        if (descriptor == null) {
            // descriptor is not defined, use default
            descriptor = TopologyDescriptor.ARBITRARY;
        }
        try {
            Criteria criteria = new Criteria(neededResourcesNumber);
            criteria.setTopology(descriptor);
            // resolve script variables (if any) in the list of selection
            // scripts and then set it as the selection criteria.
            criteria.setScripts(resolveScriptVariables(internalTask0.getSelectionScripts(), internalTask0.getRuntimeVariables()));
            criteria.setBlackList(internalTask0.getNodeExclusion());
            criteria.setBestEffort(bestEffort);
            criteria.setAcceptableNodesUrls(freeResources);
            criteria.setBindings(createBindingsForSelectionScripts(currentJob, internalTask0));
            if (internalTask0.getRuntimeGenericInformation().containsKey(SchedulerConstants.NODE_ACCESS_TOKEN)) {
                criteria.setNodeAccessToken(internalTask0.getRuntimeGenericInformation().get(SchedulerConstants.NODE_ACCESS_TOKEN));
            }
            Collection<String> computationDescriptors = new ArrayList<>(tasksToSchedule.size());
            for (EligibleTaskDescriptor task : tasksToSchedule) {
                computationDescriptors.add(TaskLogger.getTaskLogRelativePath(task.getTaskId()));
            }
            criteria.setComputationDescriptors(computationDescriptors);
            nodeSet = getRMProxiesManager().getUserRMProxy(currentJob.getOwner(), currentJob.getCredentials()).getNodes(criteria);
        } catch (TopologyDisabledException tde) {
            jlogger.warn(currentJob.getId(), "will be canceled as the topology is disabled");
            schedulingService.simulateJobStartAndCancelIt(tasksToSchedule, "Topology is disabled");
            return null;
        }
        // the following line is used to unwrap the future, warning when moving or removing
        // it may also throw a ScriptException which is a RuntimeException
        PAFuture.waitFor(nodeSet, true);
        logger.debug("provided nodes " + nodeSet.size());
        return nodeSet;
    } catch (IOException | ClassNotFoundException e) {
        logger.warn("Failed to deserialize previous task variables before selection for task " + internalTask0.getId().toString(), e);
        schedulingService.simulateJobStartAndCancelIt(tasksToSchedule, "Failed to deserialize previous task variables before selection for task " + internalTask0.getId().toString());
        return null;
    } catch (RMProxyCreationException e) {
        logger.warn("Failed to create User RM Proxy", e);
        // simulate jobs starts and cancel it
        schedulingService.simulateJobStartAndCancelIt(tasksToSchedule, "Failed to create User RM Proxy : Authentication Failed to Resource Manager for user '" + currentJob.getOwner() + "'");
        // leave the method by ss failure
        return null;
    }
}

Example 40 with SelectionScript

use of org.ow2.proactive.scripting.SelectionScript in project scheduling by ow2-proactive.

the class TaskData method createTaskData.

static TaskData createTaskData(JobData jobRuntimeData, InternalScriptTask task) {
    TaskData taskData = new TaskData();
    TaskData.DBTaskId taskId = new DBTaskId();
    taskId.setJobId(jobRuntimeData.getId());
    taskId.setTaskId(task.getTaskInfo().getTaskId().longValue());
    taskData.setId(taskId);
    taskData.setDescription(task.getDescription());
    taskData.setTag(task.getTag());
    taskData.setParallelEnvironment(task.getParallelEnvironment());
    taskData.setFlowBlock(task.getFlowBlock());
    taskData.setRestartMode(task.getRestartTaskOnError());
    taskData.setPreciousLogs(task.isPreciousLogs());
    taskData.setPreciousResult(task.isPreciousResult());
    taskData.setRunAsMe(task.isRunAsMe());
    taskData.setWallTime(task.getWallTime());
    taskData.setOnTaskErrorString(task.getOnTaskErrorProperty().getValue());
    taskData.setMaxNumberOfExecution(task.getMaxNumberOfExecution());
    taskData.setJobData(jobRuntimeData);
    taskData.setNumberOfExecutionOnFailureLeft(PASchedulerProperties.NUMBER_OF_EXECUTION_ON_FAILURE.getValueAsInt());
    taskData.setNumberOfExecutionLeft(task.getMaxNumberOfExecution());
    taskData.setGenericInformation(task.getGenericInformation());
    HashMap<String, TaskDataVariable> variables = new HashMap<>();
    for (Map.Entry<String, TaskVariable> entry : task.getVariables().entrySet()) {
        variables.put(entry.getKey(), TaskDataVariable.create(entry.getKey(), entry.getValue(), taskData));
    }
    taskData.setVariables(variables);
    // set the scheduledTime if the START_AT property exists
    Map<String, String> genericInfos = taskData.getGenericInformation();
    if (genericInfos != null && genericInfos.containsKey(CommonAttribute.GENERIC_INFO_START_AT_KEY)) {
        long scheduledTime = ISO8601DateUtil.toDate(genericInfos.get(CommonAttribute.GENERIC_INFO_START_AT_KEY)).getTime();
        taskData.setScheduledTime(scheduledTime);
        task.setScheduledTime(scheduledTime);
    }
    taskData.updateMutableAttributes(task);
    if (task.getSelectionScripts() != null) {
        List<SelectionScriptData> scripts = new ArrayList<>(task.getSelectionScripts().size());
        for (SelectionScript selectionScript : task.getSelectionScripts()) {
            scripts.add(SelectionScriptData.createForSelectionScript(selectionScript, taskData));
        }
        taskData.setSelectionScripts(scripts);
    }
    if (task.getExecutableContainer() != null) {
        taskData.setScript(ScriptData.createForScript(((ScriptExecutableContainer) task.getExecutableContainer()).getScript(), taskData));
    }
    if (task.getPreScript() != null) {
        taskData.setPreScript(ScriptData.createForScript(task.getPreScript(), taskData));
    }
    if (task.getPostScript() != null) {
        taskData.setPostScript(ScriptData.createForScript(task.getPostScript(), taskData));
    }
    if (task.getCleaningScript() != null) {
        taskData.setCleanScript(ScriptData.createForScript(task.getCleaningScript(), taskData));
    }
    if (task.getFlowScript() != null) {
        taskData.setFlowScript(ScriptData.createForFlowScript(task.getFlowScript(), taskData));
    }
    List<SelectorData> selectorsData = new ArrayList<>();
    if (task.getInputFilesList() != null) {
        for (InputSelector selector : task.getInputFilesList()) {
            selectorsData.add(SelectorData.createForInputSelector(selector, taskData));
        }
    }
    if (task.getOutputFilesList() != null) {
        for (OutputSelector selector : task.getOutputFilesList()) {
            selectorsData.add(SelectorData.createForOutputSelector(selector, taskData));
        }
    }
    taskData.setDataspaceSelectors(selectorsData);
    ForkEnvironment forkEnvironment = task.getForkEnvironment();
    if (forkEnvironment != null) {
        taskData.setAdditionalClasspath(forkEnvironment.getAdditionalClasspath());
        taskData.setJavaHome(forkEnvironment.getJavaHome());
        taskData.setJvmArguments(forkEnvironment.getJVMArguments());
        taskData.setWorkingDir(forkEnvironment.getWorkingDir());
        if (forkEnvironment.getEnvScript() != null) {
            taskData.setEnvScript(ScriptData.createForScript(forkEnvironment.getEnvScript(), taskData));
        }
        Map<String, String> systemEnvironment = forkEnvironment.getSystemEnvironment();
        if (systemEnvironment != null) {
            List<EnvironmentModifierData> envModifiers = new ArrayList<>(systemEnvironment.size());
            for (Map.Entry<String, String> entry : systemEnvironment.entrySet()) {
                envModifiers.add(EnvironmentModifierData.create(new PropertyModifier(entry.getKey(), entry.getValue()), taskData));
            }
            taskData.setEnvModifiers(envModifiers);
        }
    }
    taskData.initTaskType(task);
    return taskData;
}