Examples with ProcessModel org.apache.airavata.model.process.ProcessModel used on opensource projects

Example 1 with ProcessModel

use of org.apache.airavata.model.process.ProcessModel in project airavata by apache.

the class GFacEngineImpl method setUserComputeResourcePreference.

private void setUserComputeResourcePreference(String gatewayId, ProcessContext processContext) throws AppCatalogException {
    AppCatalog appCatalog = processContext.getAppCatalog();
    ProcessModel processModel = processContext.getProcessModel();
    UserComputeResourcePreference userComputeResourcePreference = appCatalog.getUserResourceProfile().getUserComputeResourcePreference(processModel.getUserName(), gatewayId, processModel.getComputeResourceId());
    processContext.setUserComputeResourcePreference(userComputeResourcePreference);
}

Example 2 with ProcessModel

use of org.apache.airavata.model.process.ProcessModel in project airavata by apache.

the class GFacEngineImpl method setUserResourceProfile.

private void setUserResourceProfile(String gatewayId, ProcessContext processContext) throws AppCatalogException {
    AppCatalog appCatalog = processContext.getAppCatalog();
    ProcessModel processModel = processContext.getProcessModel();
    UserResourceProfile userResourceProfile = appCatalog.getUserResourceProfile().getUserResourceProfile(processModel.getUserName(), gatewayId);
    processContext.setUserResourceProfile(userResourceProfile);
}

Example 3 with ProcessModel

use of org.apache.airavata.model.process.ProcessModel in project airavata by apache.

the class DataTransferrer method getDownloadLocation.

private String getDownloadLocation() {
    ProcessModel processModel = processContext.getProcessModel();
    String outputDataDir = "";
    if (processContext.getOutputDir() != null) {
        outputDataDir = processContext.getOutputDir();
        if ("".equals(outputDataDir)) {
            outputDataDir = getTempPath();
        } else {
            // in case of remote locations use the tmp location
            if (outputDataDir.startsWith("scp:") || outputDataDir.startsWith("ftp:") || outputDataDir.startsWith("gsiftp:")) {
                outputDataDir = getTempPath();
            } else if (outputDataDir.startsWith("file:") && outputDataDir.contains("@")) {
                outputDataDir = getTempPath();
            } else {
                try {
                    URI u = new URI(outputDataDir);
                    outputDataDir = u.getPath();
                } catch (URISyntaxException e) {
                    outputDataDir = getTempPath();
                }
            }
        }
    }
    File file = new File(outputDataDir);
    if (!file.exists()) {
        file.mkdirs();
    }
    return outputDataDir;
}

Example 4 with ProcessModel

use of org.apache.airavata.model.process.ProcessModel in project airavata by apache.

the class ResourceProcessor method generateResourceElements.

public static void generateResourceElements(JobDefinitionType value, ProcessContext context) throws Exception {
    ProcessModel processModel = context.getProcessModel();
    if (processModel != null) {
        try {
            ComputationalResourceSchedulingModel crs = processModel.getProcessResourceSchedule();
            if (crs.getTotalPhysicalMemory() > 0) {
                RangeValueType rangeType = new RangeValueType();
                rangeType.setLowerBound(Double.NaN);
                rangeType.setUpperBound(Double.NaN);
                rangeType.setExact(crs.getTotalPhysicalMemory());
                JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
            }
            if (crs.getNodeCount() > 0) {
                RangeValueType rangeType = new RangeValueType();
                rangeType.setLowerBound(Double.NaN);
                rangeType.setUpperBound(Double.NaN);
                rangeType.setExact(crs.getNodeCount());
                JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
            }
            if (crs.getWallTimeLimit() > 0) {
                RangeValueType cpuTime = new RangeValueType();
                cpuTime.setLowerBound(Double.NaN);
                cpuTime.setUpperBound(Double.NaN);
                long wallTime = crs.getWallTimeLimit() * 60;
                cpuTime.setExact(wallTime);
                JSDLUtils.setIndividualCPUTimeRequirements(value, cpuTime);
            }
            // the total cpu count is total cpus per node
            if (crs.getTotalCPUCount() > 0) {
                RangeValueType rangeType = new RangeValueType();
                rangeType.setLowerBound(Double.NaN);
                rangeType.setUpperBound(Double.NaN);
                int nodeCount = crs.getNodeCount();
                if (nodeCount <= 0) {
                    nodeCount = 1;
                }
                rangeType.setExact(crs.getTotalCPUCount() / nodeCount);
                JSDLUtils.setIndividualCPUCountRequirements(value, rangeType);
            }
            String qName = crs.getQueueName();
            if (!(qName == null || "".equals(qName))) {
                // ignore "default" queue names
                if (!(crs.getQueueName().trim().equalsIgnoreCase("default"))) {
                    ResourceRequestDocument rqDoc = ResourceRequestDocument.Factory.newInstance();
                    ResourceRequestType rq = rqDoc.addNewResourceRequest();
                    rq.setName("Queue");
                    rq.setValue(qName);
                    ResourcesType res = JSDLUtils.getOrCreateResources(value);
                    WSUtilities.insertAny(rqDoc, res);
                }
            }
        } catch (NullPointerException npe) {
            throw new Exception("No value set for resource requirements.", npe);
        }
    }
}

Example 5 with ProcessModel

use of org.apache.airavata.model.process.ProcessModel in project airavata by apache.

the class GFacUtils method createGroovyMap.

public static GroovyMap createGroovyMap(ProcessContext processContext, TaskContext taskContext) throws GFacException, AppCatalogException, ApplicationSettingsException {
    GroovyMap groovyMap = new GroovyMap();
    ProcessModel processModel = processContext.getProcessModel();
    ResourceJobManager resourceJobManager = getResourceJobManager(processContext);
    // set email options and addresses
    setMailAddresses(processContext, groovyMap);
    groovyMap.add(Script.INPUT_DIR, processContext.getInputDir());
    groovyMap.add(Script.OUTPUT_DIR, processContext.getOutputDir());
    groovyMap.add(Script.EXECUTABLE_PATH, processContext.getApplicationDeploymentDescription().getExecutablePath());
    groovyMap.add(Script.STANDARD_OUT_FILE, processContext.getStdoutLocation());
    groovyMap.add(Script.STANDARD_ERROR_FILE, processContext.getStderrLocation());
    groovyMap.add(Script.SCRATCH_LOCATION, processContext.getScratchLocation());
    groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
    groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName());
    groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName());
    groovyMap.add(Script.QUEUE_SPECIFIC_MACROS, processContext.getQueueSpecificMacros());
    groovyMap.add(Script.ACCOUNT_STRING, processContext.getAllocationProjectNumber());
    groovyMap.add(Script.RESERVATION, processContext.getReservation());
    // To make job name alpha numeric
    groovyMap.add(Script.JOB_NAME, "A" + String.valueOf(generateJobName()));
    groovyMap.add(Script.WORKING_DIR, processContext.getWorkingDir());
    List<String> inputValues = getProcessInputValues(processModel.getProcessInputs());
    inputValues.addAll(getProcessOutputValues(processModel.getProcessOutputs()));
    groovyMap.add(Script.INPUTS, inputValues);
    groovyMap.add(Script.USER_NAME, processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName());
    groovyMap.add(Script.SHELL_NAME, "/bin/bash");
    // get walltime
    if (taskContext != null) {
        try {
            JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel());
            if (jobSubmissionTaskModel.getWallTime() > 0) {
                groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
                if (resourceJobManager != null) {
                    if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                        groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(jobSubmissionTaskModel.getWallTime()));
                    }
                }
            }
        } catch (TException e) {
            log.error("Error while getting job submission sub task model", e);
        }
    }
    // NOTE: Give precedence to data comes with experiment
    // qos per queue
    String qoS = getQoS(processContext.getQualityOfService(), processContext.getQueueName());
    if (qoS != null) {
        groovyMap.add(Script.QUALITY_OF_SERVICE, qoS);
    }
    ComputationalResourceSchedulingModel scheduling = processModel.getProcessResourceSchedule();
    if (scheduling != null) {
        int totalNodeCount = scheduling.getNodeCount();
        int totalCPUCount = scheduling.getTotalCPUCount();
        if (isValid(scheduling.getQueueName())) {
            groovyMap.add(Script.QUEUE_NAME, scheduling.getQueueName());
        }
        if (totalNodeCount > 0) {
            groovyMap.add(Script.NODES, totalNodeCount);
        }
        if (totalCPUCount > 0) {
            int ppn = totalCPUCount / totalNodeCount;
            groovyMap.add(Script.PROCESS_PER_NODE, ppn);
            groovyMap.add(Script.CPU_COUNT, totalCPUCount);
        }
        // if so we ignore scheduling configuration.
        if (scheduling.getWallTimeLimit() > 0 && groovyMap.get(Script.MAX_WALL_TIME) == null) {
            groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(scheduling.getWallTimeLimit()));
            if (resourceJobManager != null) {
                if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                    groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(scheduling.getWallTimeLimit()));
                }
            }
        }
        if (scheduling.getTotalPhysicalMemory() > 0) {
            groovyMap.add(Script.USED_MEM, scheduling.getTotalPhysicalMemory());
        }
        if (isValid(scheduling.getOverrideLoginUserName())) {
            groovyMap.add(Script.USER_NAME, scheduling.getOverrideLoginUserName());
        }
        if (isValid(scheduling.getOverrideAllocationProjectNumber())) {
            groovyMap.add(Script.ACCOUNT_STRING, scheduling.getOverrideAllocationProjectNumber());
        }
        if (isValid(scheduling.getStaticWorkingDir())) {
            groovyMap.add(Script.WORKING_DIR, scheduling.getStaticWorkingDir());
        }
    } else {
        log.error("Task scheduling cannot be null at this point..");
    }
    ApplicationDeploymentDescription appDepDescription = processContext.getApplicationDeploymentDescription();
    List<CommandObject> moduleCmds = appDepDescription.getModuleLoadCmds();
    if (moduleCmds != null) {
        List<String> modulesCmdCollect = moduleCmds.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> map.getCommand()).collect(Collectors.toList());
        groovyMap.add(Script.MODULE_COMMANDS, modulesCmdCollect);
    }
    List<CommandObject> preJobCommands = appDepDescription.getPreJobCommands();
    if (preJobCommands != null) {
        List<String> preJobCmdCollect = preJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect);
    }
    List<CommandObject> postJobCommands = appDepDescription.getPostJobCommands();
    if (postJobCommands != null) {
        List<String> postJobCmdCollect = postJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect);
    }
    ApplicationParallelismType parallelism = appDepDescription.getParallelism();
    if (parallelism != null) {
        if (parallelism != ApplicationParallelismType.SERIAL) {
            Map<ApplicationParallelismType, String> parallelismPrefix = processContext.getResourceJobManager().getParallelismPrefix();
            if (parallelismPrefix != null) {
                String parallelismCommand = parallelismPrefix.get(parallelism);
                if (parallelismCommand != null) {
                    groovyMap.add(Script.JOB_SUBMITTER_COMMAND, parallelismCommand);
                } else {
                    throw new GFacException("Parallelism prefix is not defined for given parallelism type " + parallelism + ".. Please define the parallelism prefix at App Catalog");
                }
            }
        }
    }
    return groovyMap;
}