Examples with ComputationalResourceSchedulingModel org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel used on opensource projects

Example 26 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class SimpleOrchestratorImpl method createAndSaveTasks.

public String createAndSaveTasks(String gatewayId, ProcessModel processModel, boolean autoSchedule) throws OrchestratorException {
    try {
        ExperimentCatalog experimentCatalog = orchestratorContext.getRegistry().getExperimentCatalog();
        AppCatalog appCatalog = orchestratorContext.getRegistry().getAppCatalog();
        ComputationalResourceSchedulingModel resourceSchedule = processModel.getProcessResourceSchedule();
        String userGivenQueueName = resourceSchedule.getQueueName();
        int userGivenWallTime = resourceSchedule.getWallTimeLimit();
        String resourceHostId = resourceSchedule.getResourceHostId();
        if (resourceHostId == null) {
            throw new OrchestratorException("Compute Resource Id cannot be null at this point");
        }
        ComputeResourceDescription computeResource = appCatalog.getComputeResource().getComputeResource(resourceHostId);
        JobSubmissionInterface preferredJobSubmissionInterface = OrchestratorUtils.getPreferredJobSubmissionInterface(orchestratorContext, processModel, gatewayId);
        ComputeResourcePreference resourcePreference = OrchestratorUtils.getComputeResourcePreference(orchestratorContext, processModel, gatewayId);
        List<String> taskIdList = new ArrayList<>();
        if (resourcePreference.getPreferredJobSubmissionProtocol() == JobSubmissionProtocol.UNICORE) {
            // TODO - breakdown unicore all in one task to multiple tasks, then we don't need to handle UNICORE here.
            taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, preferredJobSubmissionInterface, processModel, userGivenWallTime));
        } else {
            taskIdList.addAll(createAndSaveEnvSetupTask(gatewayId, processModel, experimentCatalog));
            taskIdList.addAll(createAndSaveInputDataStagingTasks(processModel, gatewayId));
            if (autoSchedule) {
                List<BatchQueue> definedBatchQueues = computeResource.getBatchQueues();
                for (BatchQueue batchQueue : definedBatchQueues) {
                    if (batchQueue.getQueueName().equals(userGivenQueueName)) {
                        int maxRunTime = batchQueue.getMaxRunTime();
                        if (maxRunTime < userGivenWallTime) {
                            resourceSchedule.setWallTimeLimit(maxRunTime);
                            // need to create more job submissions
                            int numOfMaxWallTimeJobs = ((int) Math.floor(userGivenWallTime / maxRunTime));
                            for (int i = 1; i <= numOfMaxWallTimeJobs; i++) {
                                taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, preferredJobSubmissionInterface, processModel, maxRunTime));
                            }
                            int leftWallTime = userGivenWallTime % maxRunTime;
                            if (leftWallTime != 0) {
                                taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, preferredJobSubmissionInterface, processModel, leftWallTime));
                            }
                        } else {
                            taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, preferredJobSubmissionInterface, processModel, userGivenWallTime));
                        }
                    }
                }
            } else {
                taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, preferredJobSubmissionInterface, processModel, userGivenWallTime));
            }
            taskIdList.addAll(createAndSaveOutputDataStagingTasks(processModel, gatewayId));
        }
        // update process scheduling
        experimentCatalog.update(ExperimentCatalogModelType.PROCESS, processModel, processModel.getProcessId());
        return getTaskDag(taskIdList);
    } catch (Exception e) {
        throw new OrchestratorException("Error during creating process", e);
    }
}

Example 27 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class ThriftDataModelConversion method getComputationalResourceScheduling.

public static ComputationalResourceSchedulingModel getComputationalResourceScheduling(UserConfigurationDataResource resource) {
    if (resource != null) {
        ComputationalResourceSchedulingModel resourceSchedulingModel = new ComputationalResourceSchedulingModel();
        resourceSchedulingModel.setResourceHostId(resource.getResourceHostId());
        resourceSchedulingModel.setTotalCPUCount(resource.getTotalCpuCount());
        resourceSchedulingModel.setNodeCount(resource.getNodeCount());
        resourceSchedulingModel.setNumberOfThreads(resource.getNumberOfThreads());
        resourceSchedulingModel.setQueueName(resource.getQueueName());
        resourceSchedulingModel.setWallTimeLimit(resource.getWallTimeLimit());
        resourceSchedulingModel.setTotalPhysicalMemory(resource.getTotalPhysicalMemory());
        resourceSchedulingModel.setStaticWorkingDir(resource.getStaticWorkingDir());
        resourceSchedulingModel.setOverrideLoginUserName(resource.getOverrideLoginUserName());
        resourceSchedulingModel.setOverrideScratchLocation(resource.getOverrideScratchLocation());
        resourceSchedulingModel.setOverrideAllocationProjectNumber(resource.getOverrideAllocationProjectNumber());
        return resourceSchedulingModel;
    }
    return null;
}

Example 28 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class ThriftDataModelConversion method getUserConfigData.

public static UserConfigurationDataModel getUserConfigData(UserConfigurationDataResource resource) throws RegistryException {
    if (resource != null) {
        UserConfigurationDataModel data = new UserConfigurationDataModel();
        data.setAiravataAutoSchedule(resource.getAiravataAutoSchedule());
        data.setOverrideManualScheduledParams(resource.getOverrideManualScheduledParams());
        data.setShareExperimentPublicly(resource.getShareExperimentPublically());
        data.setUserDN(resource.getUserDn());
        data.setGenerateCert(resource.getGenerateCert());
        data.setStorageId(resource.getStorageId());
        data.setExperimentDataDir(resource.getExperimentDataDir());
        data.setUseUserCRPref(resource.getUseUserCRPref());
        ComputationalResourceSchedulingModel resourceSchedulingModel = new ComputationalResourceSchedulingModel();
        resourceSchedulingModel.setResourceHostId(resource.getResourceHostId());
        resourceSchedulingModel.setTotalCPUCount(resource.getTotalCpuCount());
        resourceSchedulingModel.setNodeCount(resource.getNodeCount());
        resourceSchedulingModel.setNumberOfThreads(resource.getNumberOfThreads());
        resourceSchedulingModel.setQueueName(resource.getQueueName());
        resourceSchedulingModel.setWallTimeLimit(resource.getWallTimeLimit());
        resourceSchedulingModel.setTotalPhysicalMemory(resource.getTotalPhysicalMemory());
        resourceSchedulingModel.setStaticWorkingDir(resource.getStaticWorkingDir());
        resourceSchedulingModel.setOverrideLoginUserName(resource.getOverrideLoginUserName());
        resourceSchedulingModel.setOverrideScratchLocation(resource.getOverrideScratchLocation());
        resourceSchedulingModel.setOverrideAllocationProjectNumber(resource.getOverrideAllocationProjectNumber());
        data.setComputationalResourceScheduling(resourceSchedulingModel);
        return data;
    }
    return null;
}

Example 29 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class CreateLaunchExperiment method createExperimentForStampedeAmber.

public static String createExperimentForStampedeAmber(Airavata.Client client) throws TException {
    try {
        List<InputDataObjectType> exInputs = client.getApplicationInputs(new AuthzToken(""), amberAppId);
        // }
        for (InputDataObjectType inputDataObjectType : exInputs) {
            if (inputDataObjectType.getName().equalsIgnoreCase("Heat_Restart_File")) {
                inputDataObjectType.setValue("file://ogce@stampede.xsede.org:/scratch/01437/ogce/gta-work-dirs/PROCESS_e0610a6c-5778-4a69-a004-f440e29194af/02_Heat.rst");
            } else if (inputDataObjectType.getName().equalsIgnoreCase("Production_Control_File")) {
                inputDataObjectType.setValue("file://ogce@stampede.xsede.org:/scratch/01437/ogce/gta-work-dirs/PROCESS_e0610a6c-5778-4a69-a004-f440e29194af/03_Prod.in");
            } else if (inputDataObjectType.getName().equalsIgnoreCase("Parameter_Topology_File")) {
                inputDataObjectType.setValue("file://ogce@stampede.xsede.org:/scratch/01437/ogce/gta-work-dirs/PROCESS_e0610a6c-5778-4a69-a004-f440e29194af/prmtop");
            }
        }
        List<OutputDataObjectType> exOut = client.getApplicationOutputs(new AuthzToken(""), amberAppId);
        Project project = ProjectModelUtil.createProject("default", "admin", "test project");
        String projectId = client.createProject(new AuthzToken(""), DEFAULT_GATEWAY, project);
        ExperimentModel simpleExperiment = ExperimentModelUtil.createSimpleExperiment(DEFAULT_GATEWAY, projectId, "admin", "sshEchoExperiment", "SimpleEchoBR", amberAppId, exInputs);
        simpleExperiment.setExperimentOutputs(exOut);
        Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(new AuthzToken(""), amberAppId);
        if (computeResources != null && computeResources.size() != 0) {
            for (String id : computeResources.keySet()) {
                String resourceName = computeResources.get(id);
                if (resourceName.equals(stampedeHostName)) {
                    ComputationalResourceSchedulingModel scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 4, 1, 1, "normal", 20, 1);
                    UserConfigurationDataModel userConfigurationData = new UserConfigurationDataModel();
                    userConfigurationData.setAiravataAutoSchedule(false);
                    userConfigurationData.setOverrideManualScheduledParams(false);
                    userConfigurationData.setComputationalResourceScheduling(scheduling);
                    simpleExperiment.setUserConfigurationData(userConfigurationData);
                    return client.createExperiment(new AuthzToken(""), DEFAULT_GATEWAY, simpleExperiment);
                }
            }
        }
    } catch (AiravataSystemException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new AiravataSystemException(e);
    } catch (InvalidRequestException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new InvalidRequestException(e);
    } catch (AiravataClientException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new AiravataClientException(e);
    } catch (TException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new TException(e);
    }
    return null;
}

Example 30 with ComputationalResourceSchedulingModel

use of org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel in project airavata by apache.

the class CreateLaunchExperiment method createMPIExperimentForFSD.

public static String createMPIExperimentForFSD(Airavata.Client client) throws TException {
    try {
        List<InputDataObjectType> exInputs = client.getApplicationInputs(new AuthzToken(""), mpiAppId);
        for (InputDataObjectType inputDataObjectType : exInputs) {
            if (inputDataObjectType.getName().equalsIgnoreCase("Sample_Input")) {
                inputDataObjectType.setValue("");
            }
        }
        List<OutputDataObjectType> exOut = client.getApplicationOutputs(new AuthzToken(""), mpiAppId);
        ExperimentModel simpleExperiment = ExperimentModelUtil.createSimpleExperiment(DEFAULT_GATEWAY, "default", "admin", "mpiExperiment", "HelloMPI", mpiAppId, null);
        simpleExperiment.setExperimentOutputs(exOut);
        Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(new AuthzToken(""), mpiAppId);
        if (computeResources != null && computeResources.size() != 0) {
            for (String id : computeResources.keySet()) {
                String resourceName = computeResources.get(id);
                if (resourceName.equals(unicoreHostName)) {
                    ComputationalResourceSchedulingModel scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 2, 1, 2, "normal", 30, 1048576);
                    UserConfigurationDataModel userConfigurationData = new UserConfigurationDataModel();
                    userConfigurationData.setAiravataAutoSchedule(false);
                    userConfigurationData.setOverrideManualScheduledParams(false);
                    userConfigurationData.setComputationalResourceScheduling(scheduling);
                    userConfigurationData.setGenerateCert(false);
                    userConfigurationData.setUserDN("");
                    simpleExperiment.setUserConfigurationData(userConfigurationData);
                    return client.createExperiment(new AuthzToken(""), DEFAULT_GATEWAY, simpleExperiment);
                }
            }
        }
    } catch (AiravataSystemException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new AiravataSystemException(e);
    } catch (InvalidRequestException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new InvalidRequestException(e);
    } catch (AiravataClientException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new AiravataClientException(e);
    } catch (TException e) {
        logger.error("Error occured while creating the experiment...", e.getMessage());
        throw new TException(e);
    }
    return null;
}