Examples with ComputeResourcePreference org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference used on opensource projects

Example 11 with ComputeResourcePreference

use of org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference in project airavata by apache.

the class RegisterSampleApplications method registerGatewayResourceProfile.

public void registerGatewayResourceProfile() {
    try {
        System.out.println("#### Registering Application Deployments on BigRed II #### \n");
        ComputeResourcePreference stampedeResourcePreferences = RegisterSampleApplicationsUtils.createComputeResourcePreference(stampedeResourceId, "TG-STA110014S", false, null, JobSubmissionProtocol.SSH, DataMovementProtocol.SCP, "/scratch/01437/ogce/gta-work-dirs");
        ComputeResourcePreference trestlesResourcePreferences = RegisterSampleApplicationsUtils.createComputeResourcePreference(trestlesResourceId, "sds128", false, null, JobSubmissionProtocol.SSH, DataMovementProtocol.SCP, "/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs");
        ComputeResourcePreference bigRedResourcePreferences = RegisterSampleApplicationsUtils.createComputeResourcePreference(bigredResourceId, "TG-STA110014S", false, null, null, null, "/N/dc2/scratch/cgateway/gta-work-dirs");
        ComputeResourcePreference lsfResourcePreferences = RegisterSampleApplicationsUtils.createComputeResourcePreference(lsfResourceId, "airavata", false, null, null, null, "/home/lg11w/mywork");
        ComputeResourcePreference fsdResourcePreferences = RegisterSampleApplicationsUtils.createComputeResourcePreference(fsdResourceId, null, false, null, JobSubmissionProtocol.UNICORE, DataMovementProtocol.UNICORE_STORAGE_SERVICE, null);
        ComputeResourcePreference alamoResourcePreferences = RegisterSampleApplicationsUtils.createComputeResourcePreference(alamoResourceId, null, false, null, JobSubmissionProtocol.SSH, DataMovementProtocol.SCP, "/home/us3/work/uslims3_cauma3-03896");
        GatewayResourceProfile gatewayResourceProfile = new GatewayResourceProfile();
        gatewayResourceProfile.setGatewayID(DEFAULT_GATEWAY);
        gatewayResourceProfile.addToComputeResourcePreferences(stampedeResourcePreferences);
        gatewayResourceProfile.addToComputeResourcePreferences(trestlesResourcePreferences);
        gatewayResourceProfile.addToComputeResourcePreferences(bigRedResourcePreferences);
        gatewayResourceProfile.addToComputeResourcePreferences(fsdResourcePreferences);
        gatewayResourceProfile.addToComputeResourcePreferences(lsfResourcePreferences);
        String gatewayProfile = airavataClient.registerGatewayResourceProfile(new AuthzToken(""), gatewayResourceProfile);
        System.out.println("Gateway Profile is registered with Id " + gatewayProfile);
    } catch (TException e) {
        e.printStackTrace();
    }
}

Example 12 with ComputeResourcePreference

use of org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference in project airavata by apache.

the class GwyResourceProfileImpl method updateGatewayResourceProfile.

@Override
public void updateGatewayResourceProfile(String gatewayId, org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile updatedProfile) throws AppCatalogException {
    try {
        GatewayProfileResource profileResource = new GatewayProfileResource();
        GatewayProfileResource existingGP = (GatewayProfileResource) profileResource.get(gatewayId);
        existingGP.setCredentialStoreToken(updatedProfile.getCredentialStoreToken());
        existingGP.setIdentityServerTenant(updatedProfile.getIdentityServerTenant());
        existingGP.setIdentityServerPwdCredToken(updatedProfile.getIdentityServerPwdCredToken());
        existingGP.save();
        List<ComputeResourcePreference> computeResourcePreferences = updatedProfile.getComputeResourcePreferences();
        if (computeResourcePreferences != null && !computeResourcePreferences.isEmpty()) {
            for (ComputeResourcePreference preference : computeResourcePreferences) {
                ComputeHostPreferenceResource resource = new ComputeHostPreferenceResource();
                resource.setGatewayProfile(existingGP);
                resource.setResourceId(preference.getComputeResourceId());
                ComputeResourceResource computeHostResource = new ComputeResourceResource();
                resource.setComputeHostResource((ComputeResourceResource) computeHostResource.get(preference.getComputeResourceId()));
                resource.setGatewayId(gatewayId);
                resource.setLoginUserName(preference.getLoginUserName());
                resource.setOverrideByAiravata(preference.isOverridebyAiravata());
                if (preference.getPreferredJobSubmissionProtocol() != null) {
                    resource.setPreferredJobProtocol(preference.getPreferredJobSubmissionProtocol().toString());
                }
                if (preference.getPreferredDataMovementProtocol() != null) {
                    resource.setPreferedDMProtocol(preference.getPreferredDataMovementProtocol().toString());
                }
                resource.setBatchQueue(preference.getPreferredBatchQueue());
                resource.setProjectNumber(preference.getAllocationProjectNumber());
                resource.setScratchLocation(preference.getScratchLocation());
                resource.setResourceCSToken(preference.getResourceSpecificCredentialStoreToken());
                resource.setUsageReportingGatewayId(preference.getUsageReportingGatewayId());
                resource.setQualityOfService(preference.getQualityOfService());
                resource.setReservation(preference.getReservation());
                if (preference.getReservationStartTime() > 0) {
                    resource.setReservationStartTime(AiravataUtils.getTime(preference.getReservationStartTime()));
                }
                if (preference.getReservationEndTime() > 0) {
                    resource.setReservationEndTime(AiravataUtils.getTime(preference.getReservationEndTime()));
                }
                resource.setSshAccountProvisioner(preference.getSshAccountProvisioner());
                if (preference.getSshAccountProvisionerConfig() != null && !preference.getSshAccountProvisionerConfig().isEmpty()) {
                    Map<String, String> sshAccountProvisionerConfigurationsCopy = new HashMap<>(preference.getSshAccountProvisionerConfig());
                    resource.setSshAccountProvisionerConfigurations(sshAccountProvisionerConfigurationsCopy);
                }
                resource.setSshAccountProvisionerAdditionalInfo(preference.getSshAccountProvisionerAdditionalInfo());
                resource.save();
            }
        }
        List<StoragePreference> dataStoragePreferences = updatedProfile.getStoragePreferences();
        if (dataStoragePreferences != null && !dataStoragePreferences.isEmpty()) {
            for (StoragePreference storagePreference : dataStoragePreferences) {
                StoragePreferenceResource resource = new StoragePreferenceResource();
                resource.setStorageResourceId(storagePreference.getStorageResourceId());
                resource.setGatewayId(existingGP.getGatewayID());
                resource.setFsRootLocation(storagePreference.getFileSystemRootLocation());
                resource.setLoginUserName(storagePreference.getLoginUserName());
                resource.setResourceCSToken(storagePreference.getResourceSpecificCredentialStoreToken());
                resource.setGatewayProfile(existingGP);
                resource.save();
            }
        }
    } catch (Exception e) {
        logger.error("Error while updating gateway profile...", e);
        throw new AppCatalogException(e);
    }
}

Example 13 with ComputeResourcePreference

use of org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference in project airavata by apache.

the class OrchestratorServerHandler method launchExperiment.

/**
 * * After creating the experiment Data user have the * experimentID as the
 * handler to the experiment, during the launchProcess * We just have to
 * give the experimentID * * @param experimentID * @return sucess/failure *
 * *
 *
 * @param experimentId
 */
public boolean launchExperiment(String experimentId, String gatewayId) throws TException {
    ExperimentModel experiment = null;
    try {
        String experimentNodePath = GFacUtils.getExperimentNodePath(experimentId);
        ZKPaths.mkdirs(curatorClient.getZookeeperClient().getZooKeeper(), experimentNodePath);
        String experimentCancelNode = ZKPaths.makePath(experimentNodePath, ZkConstants.ZOOKEEPER_CANCEL_LISTENER_NODE);
        ZKPaths.mkdirs(curatorClient.getZookeeperClient().getZooKeeper(), experimentCancelNode);
        experiment = (ExperimentModel) experimentCatalog.get(ExperimentCatalogModelType.EXPERIMENT, experimentId);
        if (experiment == null) {
            log.error("Error retrieving the Experiment by the given experimentID: {} ", experimentId);
            return false;
        }
        ComputeResourcePreference computeResourcePreference = appCatalog.getGatewayProfile().getComputeResourcePreference(gatewayId, experiment.getUserConfigurationData().getComputationalResourceScheduling().getResourceHostId());
        String token = computeResourcePreference.getResourceSpecificCredentialStoreToken();
        if (token == null || token.isEmpty()) {
            // try with gateway profile level token
            GatewayResourceProfile gatewayProfile = appCatalog.getGatewayProfile().getGatewayProfile(gatewayId);
            token = gatewayProfile.getCredentialStoreToken();
        }
        // still the token is empty, then we fail the experiment
        if (token == null || token.isEmpty()) {
            log.error("You have not configured credential store token at gateway profile or compute resource preference." + " Please provide the correct token at gateway profile or compute resource preference.");
            return false;
        }
        ExperimentType executionType = experiment.getExperimentType();
        if (executionType == ExperimentType.SINGLE_APPLICATION) {
            // its an single application execution experiment
            List<ProcessModel> processes = orchestrator.createProcesses(experimentId, gatewayId);
            for (ProcessModel processModel : processes) {
                // FIXME Resolving replica if available. This is a very crude way of resolving input replicas. A full featured
                // FIXME replica resolving logic should come here
                ReplicaCatalog replicaCatalog = RegistryFactory.getReplicaCatalog();
                processModel.getProcessInputs().stream().forEach(pi -> {
                    if (pi.getType().equals(DataType.URI) && pi.getValue().startsWith("airavata-dp://")) {
                        try {
                            DataProductModel dataProductModel = replicaCatalog.getDataProduct(pi.getValue());
                            Optional<DataReplicaLocationModel> rpLocation = dataProductModel.getReplicaLocations().stream().filter(rpModel -> rpModel.getReplicaLocationCategory().equals(ReplicaLocationCategory.GATEWAY_DATA_STORE)).findFirst();
                            if (rpLocation.isPresent()) {
                                pi.setValue(rpLocation.get().getFilePath());
                                pi.setStorageResourceId(rpLocation.get().getStorageResourceId());
                            } else {
                                log.error("Could not find a replica for the URI " + pi.getValue());
                            }
                        } catch (ReplicaCatalogException e) {
                            log.error(e.getMessage(), e);
                        }
                    } else if (pi.getType().equals(DataType.URI_COLLECTION) && pi.getValue().contains("airavata-dp://")) {
                        try {
                            String[] uriList = pi.getValue().split(",");
                            final ArrayList<String> filePathList = new ArrayList<>();
                            for (String uri : uriList) {
                                if (uri.startsWith("airavata-dp://")) {
                                    DataProductModel dataProductModel = replicaCatalog.getDataProduct(uri);
                                    Optional<DataReplicaLocationModel> rpLocation = dataProductModel.getReplicaLocations().stream().filter(rpModel -> rpModel.getReplicaLocationCategory().equals(ReplicaLocationCategory.GATEWAY_DATA_STORE)).findFirst();
                                    if (rpLocation.isPresent()) {
                                        filePathList.add(rpLocation.get().getFilePath());
                                    } else {
                                        log.error("Could not find a replica for the URI " + pi.getValue());
                                    }
                                } else {
                                    // uri is in file path format
                                    filePathList.add(uri);
                                }
                            }
                            pi.setValue(StringUtils.join(filePathList, ','));
                        } catch (ReplicaCatalogException e) {
                            log.error(e.getMessage(), e);
                        }
                    }
                });
                String taskDag = orchestrator.createAndSaveTasks(gatewayId, processModel, experiment.getUserConfigurationData().isAiravataAutoSchedule());
                processModel.setTaskDag(taskDag);
                experimentCatalog.update(ExperimentCatalogModelType.PROCESS, processModel, processModel.getProcessId());
            }
            if (!validateProcess(experimentId, processes)) {
                log.error("Validating process fails for given experiment Id : {}", experimentId);
                return false;
            }
            log.debug(experimentId, "Launching single application experiment {}.", experimentId);
            ExperimentStatus status = new ExperimentStatus(ExperimentState.LAUNCHED);
            status.setReason("submitted all processes");
            status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
            OrchestratorUtils.updageAndPublishExperimentStatus(experimentId, status, publisher, gatewayId);
            log.info("expId: {}, Launched experiment ", experimentId);
            OrchestratorServerThreadPoolExecutor.getCachedThreadPool().execute(MDCUtil.wrapWithMDC(new SingleAppExperimentRunner(experimentId, token, gatewayId)));
        } else if (executionType == ExperimentType.WORKFLOW) {
            // its a workflow execution experiment
            log.debug(experimentId, "Launching workflow experiment {}.", experimentId);
            launchWorkflowExperiment(experimentId, token, gatewayId);
        } else {
            log.error(experimentId, "Couldn't identify experiment type, experiment {} is neither single application nor workflow.", experimentId);
            throw new TException("Experiment '" + experimentId + "' launch failed. Unable to figureout execution type for application " + experiment.getExecutionId());
        }
    } catch (LaunchValidationException launchValidationException) {
        ExperimentStatus status = new ExperimentStatus(ExperimentState.FAILED);
        status.setReason("Validation failed: " + launchValidationException.getErrorMessage());
        status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
        OrchestratorUtils.updageAndPublishExperimentStatus(experimentId, status, publisher, gatewayId);
        throw new TException("Experiment '" + experimentId + "' launch failed. Experiment failed to validate: " + launchValidationException.getErrorMessage(), launchValidationException);
    } catch (Exception e) {
        throw new TException("Experiment '" + experimentId + "' launch failed. Unable to figureout execution type for application " + experiment.getExecutionId(), e);
    }
    return true;
}

Example 14 with ComputeResourcePreference

use of org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference in project airavata by apache.

the class OrchestratorUtils method getLoginUserName.

public static String getLoginUserName(OrchestratorContext context, ProcessModel processModel, String gatewayId) throws RegistryException, AiravataException {
    try {
        ComputeResourcePreference computeResourcePreference = getComputeResourcePreference(context, gatewayId, processModel.getComputeResourceId());
        ComputationalResourceSchedulingModel processResourceSchedule = processModel.getProcessResourceSchedule();
        if (processModel.isUseUserCRPref()) {
            UsrResourceProfile userResourceProfile = getUserResourceProfile(context);
            UserComputeResourcePreference userComputeResourcePreference = userResourceProfile.getUserComputeResourcePreference(processModel.getUserName(), gatewayId, processModel.getComputeResourceId());
            if (isValid(userComputeResourcePreference.getLoginUserName())) {
                return userComputeResourcePreference.getLoginUserName();
            } else if (isValid(processResourceSchedule.getOverrideLoginUserName())) {
                logger.warn("User computer resource preference doesn't have valid user login name, using computer " + "resource scheduling login name " + processResourceSchedule.getOverrideLoginUserName());
                return processResourceSchedule.getOverrideLoginUserName();
            } else if (isValid(computeResourcePreference.getLoginUserName())) {
                logger.warn("Either User computer resource preference or computer resource scheduling " + "doesn't have valid user login name, using  gateway computer resource preference login name " + computeResourcePreference.getLoginUserName());
                return computeResourcePreference.getLoginUserName();
            } else {
                throw new AiravataException("Login name is not found");
            }
        } else {
            if (isValid(processResourceSchedule.getOverrideLoginUserName())) {
                return processResourceSchedule.getOverrideLoginUserName();
            } else if (isValid(computeResourcePreference.getLoginUserName())) {
                logger.warn("Process compute resource scheduling doesn't have valid user login name, " + "using  gateway computer resource preference login name " + computeResourcePreference.getLoginUserName());
                return computeResourcePreference.getLoginUserName();
            } else {
                throw new AiravataException("Login name is not found");
            }
        }
    } catch (AppCatalogException e) {
        logger.error("Error occurred while initializing app catalog to fetch login username", e);
        throw new RegistryException("Error occurred while initializing app catalog to fetch login username", e);
    }
}

Example 15 with ComputeResourcePreference

use of org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference in project airavata by apache.

the class OrchestratorUtils method getPreferredJobSubmissionInterface.

public static JobSubmissionInterface getPreferredJobSubmissionInterface(OrchestratorContext context, ProcessModel processModel, String gatewayId) throws RegistryException {
    try {
        String resourceHostId = processModel.getComputeResourceId();
        ComputeResourcePreference resourcePreference = getComputeResourcePreference(context, processModel, gatewayId);
        JobSubmissionProtocol preferredJobSubmissionProtocol = resourcePreference.getPreferredJobSubmissionProtocol();
        ComputeResourceDescription resourceDescription = context.getRegistry().getAppCatalog().getComputeResource().getComputeResource(resourceHostId);
        List<JobSubmissionInterface> jobSubmissionInterfaces = resourceDescription.getJobSubmissionInterfaces();
        Map<JobSubmissionProtocol, List<JobSubmissionInterface>> orderedInterfaces = new HashMap<>();
        List<JobSubmissionInterface> interfaces = new ArrayList<>();
        if (jobSubmissionInterfaces != null && !jobSubmissionInterfaces.isEmpty()) {
            for (JobSubmissionInterface submissionInterface : jobSubmissionInterfaces) {
                if (preferredJobSubmissionProtocol != null) {
                    if (preferredJobSubmissionProtocol.toString().equals(submissionInterface.getJobSubmissionProtocol().toString())) {
                        if (orderedInterfaces.containsKey(submissionInterface.getJobSubmissionProtocol())) {
                            List<JobSubmissionInterface> interfaceList = orderedInterfaces.get(submissionInterface.getJobSubmissionProtocol());
                            interfaceList.add(submissionInterface);
                        } else {
                            interfaces.add(submissionInterface);
                            orderedInterfaces.put(submissionInterface.getJobSubmissionProtocol(), interfaces);
                        }
                    }
                } else {
                    Collections.sort(jobSubmissionInterfaces, (jobSubmissionInterface, jobSubmissionInterface2) -> jobSubmissionInterface.getPriorityOrder() - jobSubmissionInterface2.getPriorityOrder());
                }
            }
            interfaces = orderedInterfaces.get(preferredJobSubmissionProtocol);
            Collections.sort(interfaces, (jobSubmissionInterface, jobSubmissionInterface2) -> jobSubmissionInterface.getPriorityOrder() - jobSubmissionInterface2.getPriorityOrder());
        } else {
            throw new RegistryException("Compute resource should have at least one job submission interface defined...");
        }
        return interfaces.get(0);
    } catch (AppCatalogException e) {
        throw new RegistryException("Error occurred while retrieving data from app catalog", e);
    }
}