Examples with ComputationalResourceSchedulingModel org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel used on opensource projects

Example 1 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class DataStreamingTask method execute.

@Override
public TaskStatus execute(TaskContext taskContext) {
    ProcessState processState = taskContext.getParentProcessContext().getProcessState();
    try {
        TaskStatus status = new TaskStatus(TaskState.EXECUTING);
        final DataStagingTaskModel subTaskModel = (DataStagingTaskModel) ThriftUtils.getSubTaskModel(taskContext.getTaskModel());
        if (processState == ProcessState.OUTPUT_DATA_STAGING) {
            OutputDataObjectType processOutput = taskContext.getProcessOutput();
            if (processOutput != null && processOutput.getValue() == null) {
                log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , file name shouldn't be null", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), processOutput.getName());
                status = new TaskStatus(TaskState.FAILED);
                if (processOutput.isIsRequired()) {
                    status.setReason("File name is null, but this output's isRequired bit is not set");
                } else {
                    status.setReason("File name is null");
                }
                return status;
            }
            if (processOutput != null) {
                if (processOutput.isOutputStreaming()) {
                    // stream output periodically
                    ComputationalResourceSchedulingModel resourceSchedule = taskContext.getParentProcessContext().getProcessModel().getProcessResourceSchedule();
                    int wallTimeLimit = resourceSchedule.getWallTimeLimit();
                    if (wallTimeLimit > 10) {
                        int period = wallTimeLimit / 10;
                        Timer timer = new Timer();
                        StreamData streamData = new StreamData(userName, hostName, inputPath, taskContext, subTaskModel);
                        timer.schedule(streamData, 0, 1000 * 60 * period);
                        status.setState(TaskState.COMPLETED);
                    }
                }
            }
        }
        return null;
    } catch (TException e) {
        log.error("Error while creating data streaming task", e);
        return null;
    }
}

Example 2 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class ResourceProcessor method generateResourceElements.

public static void generateResourceElements(JobDefinitionType value, ProcessContext context) throws Exception {
    ProcessModel processModel = context.getProcessModel();
    if (processModel != null) {
        try {
            ComputationalResourceSchedulingModel crs = processModel.getProcessResourceSchedule();
            if (crs.getTotalPhysicalMemory() > 0) {
                RangeValueType rangeType = new RangeValueType();
                rangeType.setLowerBound(Double.NaN);
                rangeType.setUpperBound(Double.NaN);
                rangeType.setExact(crs.getTotalPhysicalMemory());
                JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
            }
            if (crs.getNodeCount() > 0) {
                RangeValueType rangeType = new RangeValueType();
                rangeType.setLowerBound(Double.NaN);
                rangeType.setUpperBound(Double.NaN);
                rangeType.setExact(crs.getNodeCount());
                JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
            }
            if (crs.getWallTimeLimit() > 0) {
                RangeValueType cpuTime = new RangeValueType();
                cpuTime.setLowerBound(Double.NaN);
                cpuTime.setUpperBound(Double.NaN);
                long wallTime = crs.getWallTimeLimit() * 60;
                cpuTime.setExact(wallTime);
                JSDLUtils.setIndividualCPUTimeRequirements(value, cpuTime);
            }
            // the total cpu count is total cpus per node
            if (crs.getTotalCPUCount() > 0) {
                RangeValueType rangeType = new RangeValueType();
                rangeType.setLowerBound(Double.NaN);
                rangeType.setUpperBound(Double.NaN);
                int nodeCount = crs.getNodeCount();
                if (nodeCount <= 0) {
                    nodeCount = 1;
                }
                rangeType.setExact(crs.getTotalCPUCount() / nodeCount);
                JSDLUtils.setIndividualCPUCountRequirements(value, rangeType);
            }
            String qName = crs.getQueueName();
            if (!(qName == null || "".equals(qName))) {
                // ignore "default" queue names
                if (!(crs.getQueueName().trim().equalsIgnoreCase("default"))) {
                    ResourceRequestDocument rqDoc = ResourceRequestDocument.Factory.newInstance();
                    ResourceRequestType rq = rqDoc.addNewResourceRequest();
                    rq.setName("Queue");
                    rq.setValue(qName);
                    ResourcesType res = JSDLUtils.getOrCreateResources(value);
                    WSUtilities.insertAny(rqDoc, res);
                }
            }
        } catch (NullPointerException npe) {
            throw new Exception("No value set for resource requirements.", npe);
        }
    }
}

Example 3 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class ExperimentExecution method createLocalEchoExperiment.

public String createLocalEchoExperiment(String gatewayId, String applicationInterfaceId, String storageId, String computeResourceId) throws Exception {
    String experimentId = null;
    try {
        List<InputDataObjectType> applicationInputs = airavata.getApplicationInputs(authzToken, applicationInterfaceId);
        List<OutputDataObjectType> appOutputs = airavata.getApplicationOutputs(authzToken, applicationInterfaceId);
        for (InputDataObjectType inputDataObjectType : applicationInputs) {
            if (inputDataObjectType.getName().equalsIgnoreCase(INPUT_NAME)) {
                inputDataObjectType.setValue(LOCAL_ECHO_EXPERIMENT_INPUT);
            }
        }
        List<Project> projectsPerGateway = projectsMap.get(gatewayId);
        String projectID = null;
        if (projectsPerGateway != null && !projectsPerGateway.isEmpty()) {
            projectID = projectsPerGateway.get(0).getProjectID();
        }
        ExperimentModel simpleExperiment = ExperimentModelUtil.createSimpleExperiment(gatewayId, projectID, properties.getTestUserName(), "Local Echo Experiment", "Local Echo Experiment run", applicationInterfaceId, applicationInputs);
        simpleExperiment.setExperimentOutputs(appOutputs);
        ComputationalResourceSchedulingModel scheduling = ExperimentModelUtil.createComputationResourceScheduling(computeResourceId, 4, 1, 1, "cpu", 20, 0);
        UserConfigurationDataModel userConfigurationData = new UserConfigurationDataModel();
        userConfigurationData.setAiravataAutoSchedule(false);
        userConfigurationData.setOverrideManualScheduledParams(false);
        userConfigurationData.setComputationalResourceScheduling(scheduling);
        userConfigurationData.setStorageId(storageId);
        userConfigurationData.setExperimentDataDir(TestFrameworkConstants.STORAGE_LOCATION);
        simpleExperiment.setUserConfigurationData(userConfigurationData);
        experimentId = airavata.createExperiment(authzToken, gatewayId, simpleExperiment);
        experimentsWithGateway.put(experimentId, gatewayId);
    } catch (Exception e) {
        logger.error("Error while creating Echo experiment", e);
        throw new Exception("Error while creating Echo experiment", e);
    }
    return experimentId;
}

Example 4 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class GFacUtils method createGroovyMap.

public static GroovyMap createGroovyMap(ProcessContext processContext, TaskContext taskContext) throws GFacException, AppCatalogException, ApplicationSettingsException {
    GroovyMap groovyMap = new GroovyMap();
    ProcessModel processModel = processContext.getProcessModel();
    ResourceJobManager resourceJobManager = getResourceJobManager(processContext);
    // set email options and addresses
    setMailAddresses(processContext, groovyMap);
    groovyMap.add(Script.INPUT_DIR, processContext.getInputDir());
    groovyMap.add(Script.OUTPUT_DIR, processContext.getOutputDir());
    groovyMap.add(Script.EXECUTABLE_PATH, processContext.getApplicationDeploymentDescription().getExecutablePath());
    groovyMap.add(Script.STANDARD_OUT_FILE, processContext.getStdoutLocation());
    groovyMap.add(Script.STANDARD_ERROR_FILE, processContext.getStderrLocation());
    groovyMap.add(Script.SCRATCH_LOCATION, processContext.getScratchLocation());
    groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
    groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName());
    groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName());
    groovyMap.add(Script.QUEUE_SPECIFIC_MACROS, processContext.getQueueSpecificMacros());
    groovyMap.add(Script.ACCOUNT_STRING, processContext.getAllocationProjectNumber());
    groovyMap.add(Script.RESERVATION, processContext.getReservation());
    // To make job name alpha numeric
    groovyMap.add(Script.JOB_NAME, "A" + String.valueOf(generateJobName()));
    groovyMap.add(Script.WORKING_DIR, processContext.getWorkingDir());
    List<String> inputValues = getProcessInputValues(processModel.getProcessInputs());
    inputValues.addAll(getProcessOutputValues(processModel.getProcessOutputs()));
    groovyMap.add(Script.INPUTS, inputValues);
    groovyMap.add(Script.USER_NAME, processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName());
    groovyMap.add(Script.SHELL_NAME, "/bin/bash");
    // get walltime
    if (taskContext != null) {
        try {
            JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel());
            if (jobSubmissionTaskModel.getWallTime() > 0) {
                groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
                if (resourceJobManager != null) {
                    if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                        groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(jobSubmissionTaskModel.getWallTime()));
                    }
                }
            }
        } catch (TException e) {
            log.error("Error while getting job submission sub task model", e);
        }
    }
    // NOTE: Give precedence to data comes with experiment
    // qos per queue
    String qoS = getQoS(processContext.getQualityOfService(), processContext.getQueueName());
    if (qoS != null) {
        groovyMap.add(Script.QUALITY_OF_SERVICE, qoS);
    }
    ComputationalResourceSchedulingModel scheduling = processModel.getProcessResourceSchedule();
    if (scheduling != null) {
        int totalNodeCount = scheduling.getNodeCount();
        int totalCPUCount = scheduling.getTotalCPUCount();
        if (isValid(scheduling.getQueueName())) {
            groovyMap.add(Script.QUEUE_NAME, scheduling.getQueueName());
        }
        if (totalNodeCount > 0) {
            groovyMap.add(Script.NODES, totalNodeCount);
        }
        if (totalCPUCount > 0) {
            int ppn = totalCPUCount / totalNodeCount;
            groovyMap.add(Script.PROCESS_PER_NODE, ppn);
            groovyMap.add(Script.CPU_COUNT, totalCPUCount);
        }
        // if so we ignore scheduling configuration.
        if (scheduling.getWallTimeLimit() > 0 && groovyMap.get(Script.MAX_WALL_TIME) == null) {
            groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(scheduling.getWallTimeLimit()));
            if (resourceJobManager != null) {
                if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                    groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(scheduling.getWallTimeLimit()));
                }
            }
        }
        if (scheduling.getTotalPhysicalMemory() > 0) {
            groovyMap.add(Script.USED_MEM, scheduling.getTotalPhysicalMemory());
        }
        if (isValid(scheduling.getOverrideLoginUserName())) {
            groovyMap.add(Script.USER_NAME, scheduling.getOverrideLoginUserName());
        }
        if (isValid(scheduling.getOverrideAllocationProjectNumber())) {
            groovyMap.add(Script.ACCOUNT_STRING, scheduling.getOverrideAllocationProjectNumber());
        }
        if (isValid(scheduling.getStaticWorkingDir())) {
            groovyMap.add(Script.WORKING_DIR, scheduling.getStaticWorkingDir());
        }
    } else {
        log.error("Task scheduling cannot be null at this point..");
    }
    ApplicationDeploymentDescription appDepDescription = processContext.getApplicationDeploymentDescription();
    List<CommandObject> moduleCmds = appDepDescription.getModuleLoadCmds();
    if (moduleCmds != null) {
        List<String> modulesCmdCollect = moduleCmds.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> map.getCommand()).collect(Collectors.toList());
        groovyMap.add(Script.MODULE_COMMANDS, modulesCmdCollect);
    }
    List<CommandObject> preJobCommands = appDepDescription.getPreJobCommands();
    if (preJobCommands != null) {
        List<String> preJobCmdCollect = preJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect);
    }
    List<CommandObject> postJobCommands = appDepDescription.getPostJobCommands();
    if (postJobCommands != null) {
        List<String> postJobCmdCollect = postJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect);
    }
    ApplicationParallelismType parallelism = appDepDescription.getParallelism();
    if (parallelism != null) {
        if (parallelism != ApplicationParallelismType.SERIAL) {
            Map<ApplicationParallelismType, String> parallelismPrefix = processContext.getResourceJobManager().getParallelismPrefix();
            if (parallelismPrefix != null) {
                String parallelismCommand = parallelismPrefix.get(parallelism);
                if (parallelismCommand != null) {
                    groovyMap.add(Script.JOB_SUBMITTER_COMMAND, parallelismCommand);
                } else {
                    throw new GFacException("Parallelism prefix is not defined for given parallelism type " + parallelism + ".. Please define the parallelism prefix at App Catalog");
                }
            }
        }
    }
    return groovyMap;
}

Example 5 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class SimpleEchoIT method testSimpleLocalhostEchoService.

@Test
public void testSimpleLocalhostEchoService() throws Exception {
    log.info("Running job in localhost");
    log.info("========================");
    log.info("Adding applications...");
    DocumentCreatorNew documentCreatorNew = new DocumentCreatorNew(getClient());
    String hostAndappId = documentCreatorNew.createLocalHostDocs();
    String appId = hostAndappId.split(",")[1];
    String hostId = hostAndappId.split(",")[0];
    List<InputDataObjectType> exInputs = new ArrayList<InputDataObjectType>();
    InputDataObjectType input = new InputDataObjectType();
    input.setName("echo_input");
    input.setType(DataType.STRING);
    input.setValue("echo_output=Hello World");
    exInputs.add(input);
    List<OutputDataObjectType> exOut = new ArrayList<OutputDataObjectType>();
    OutputDataObjectType output = new OutputDataObjectType();
    output.setName("echo_output");
    output.setType(DataType.STRING);
    output.setValue("");
    exOut.add(output);
    Project project = ProjectModelUtil.createProject("project1", "admin", "test project");
    String projectId = getClient().createProject(authzToken, "php_reference_gateway", project);
    ExperimentModel simpleExperiment = ExperimentModelUtil.createSimpleExperiment("php_reference_gateway", projectId, "admin", "echoExperiment", appId, appId, exInputs);
    simpleExperiment.setExperimentOutputs(exOut);
    ComputationalResourceSchedulingModel scheduling = ExperimentModelUtil.createComputationResourceScheduling(hostId, 1, 1, 1, "normal", 1, 0);
    scheduling.setResourceHostId(hostId);
    UserConfigurationDataModel userConfigurationData = new UserConfigurationDataModel();
    userConfigurationData.setAiravataAutoSchedule(false);
    userConfigurationData.setOverrideManualScheduledParams(false);
    userConfigurationData.setComputationalResourceScheduling(scheduling);
    simpleExperiment.setUserConfigurationData(userConfigurationData);
    log.info("Creating experiment...");
    final String expId = createExperiment(simpleExperiment);
    log.info("Experiment Id returned : " + expId);
    launchExperiment(expId);
    log.info("Experiment launched successfully\n");
    log.info("Monitoring job in localhost");
    log.info("===========================");
    monitorJob(expId);
}