Examples with ExperimentModel org.apache.airavata.model.experiment.ExperimentModel used on opensource projects

Example 21 with ExperimentModel

use of org.apache.airavata.model.experiment.ExperimentModel in project airavata by apache.

the class OrchestratorServerHandler method launchExperiment.

private void launchExperiment(MessageContext messageContext) {
    ExperimentSubmitEvent expEvent = new ExperimentSubmitEvent();
    try {
        byte[] bytes = ThriftUtils.serializeThriftObject(messageContext.getEvent());
        ThriftUtils.createThriftFromBytes(bytes, expEvent);
        MDC.put(MDCConstants.EXPERIMENT_ID, expEvent.getExperimentId());
        log.info("Launching experiment with experimentId: {} gateway Id: {}", expEvent.getExperimentId(), expEvent.getGatewayId());
        if (messageContext.isRedeliver()) {
            ExperimentModel experimentModel = (ExperimentModel) experimentCatalog.get(ExperimentCatalogModelType.EXPERIMENT, expEvent.getExperimentId());
            MDC.put(MDCConstants.EXPERIMENT_NAME, experimentModel.getExperimentName());
            if (experimentModel.getExperimentStatus().get(0).getState() == ExperimentState.CREATED) {
                launchExperiment(expEvent.getExperimentId(), expEvent.getGatewayId());
            }
        } else {
            launchExperiment(expEvent.getExperimentId(), expEvent.getGatewayId());
        }
    } catch (TException e) {
        String logMessage = expEvent.getExperimentId() != null && expEvent.getGatewayId() != null ? String.format("Experiment launch failed due to Thrift conversion error, experimentId: %s, gatewayId: %s", expEvent.getExperimentId(), expEvent.getGatewayId()) : "Experiment launch failed due to Thrift conversion error";
        log.error(logMessage, e);
    } catch (RegistryException e) {
        String logMessage = expEvent.getExperimentId() != null && expEvent.getGatewayId() != null ? String.format("Experiment launch failed due to registry access issue, experimentId: %s, gatewayId: %s", expEvent.getExperimentId(), expEvent.getGatewayId()) : "Experiment launch failed due to registry access issue";
        log.error(logMessage, e);
    } finally {
        experimentSubscriber.sendAck(messageContext.getDeliveryTag());
        MDC.clear();
    }
}

Example 22 with ExperimentModel

use of org.apache.airavata.model.experiment.ExperimentModel in project airavata by apache.

the class OrchestratorServerHandler method launchExperiment.

/**
 * * After creating the experiment Data user have the * experimentID as the
 * handler to the experiment, during the launchProcess * We just have to
 * give the experimentID * * @param experimentID * @return sucess/failure *
 * *
 *
 * @param experimentId
 */
public boolean launchExperiment(String experimentId, String gatewayId) throws TException {
    ExperimentModel experiment = null;
    try {
        String experimentNodePath = GFacUtils.getExperimentNodePath(experimentId);
        ZKPaths.mkdirs(curatorClient.getZookeeperClient().getZooKeeper(), experimentNodePath);
        String experimentCancelNode = ZKPaths.makePath(experimentNodePath, ZkConstants.ZOOKEEPER_CANCEL_LISTENER_NODE);
        ZKPaths.mkdirs(curatorClient.getZookeeperClient().getZooKeeper(), experimentCancelNode);
        experiment = (ExperimentModel) experimentCatalog.get(ExperimentCatalogModelType.EXPERIMENT, experimentId);
        if (experiment == null) {
            log.error("Error retrieving the Experiment by the given experimentID: {} ", experimentId);
            return false;
        }
        ComputeResourcePreference computeResourcePreference = appCatalog.getGatewayProfile().getComputeResourcePreference(gatewayId, experiment.getUserConfigurationData().getComputationalResourceScheduling().getResourceHostId());
        String token = computeResourcePreference.getResourceSpecificCredentialStoreToken();
        if (token == null || token.isEmpty()) {
            // try with gateway profile level token
            GatewayResourceProfile gatewayProfile = appCatalog.getGatewayProfile().getGatewayProfile(gatewayId);
            token = gatewayProfile.getCredentialStoreToken();
        }
        // still the token is empty, then we fail the experiment
        if (token == null || token.isEmpty()) {
            log.error("You have not configured credential store token at gateway profile or compute resource preference." + " Please provide the correct token at gateway profile or compute resource preference.");
            return false;
        }
        ExperimentType executionType = experiment.getExperimentType();
        if (executionType == ExperimentType.SINGLE_APPLICATION) {
            // its an single application execution experiment
            List<ProcessModel> processes = orchestrator.createProcesses(experimentId, gatewayId);
            for (ProcessModel processModel : processes) {
                // FIXME Resolving replica if available. This is a very crude way of resolving input replicas. A full featured
                // FIXME replica resolving logic should come here
                ReplicaCatalog replicaCatalog = RegistryFactory.getReplicaCatalog();
                processModel.getProcessInputs().stream().forEach(pi -> {
                    if (pi.getType().equals(DataType.URI) && pi.getValue().startsWith("airavata-dp://")) {
                        try {
                            DataProductModel dataProductModel = replicaCatalog.getDataProduct(pi.getValue());
                            Optional<DataReplicaLocationModel> rpLocation = dataProductModel.getReplicaLocations().stream().filter(rpModel -> rpModel.getReplicaLocationCategory().equals(ReplicaLocationCategory.GATEWAY_DATA_STORE)).findFirst();
                            if (rpLocation.isPresent()) {
                                pi.setValue(rpLocation.get().getFilePath());
                                pi.setStorageResourceId(rpLocation.get().getStorageResourceId());
                            } else {
                                log.error("Could not find a replica for the URI " + pi.getValue());
                            }
                        } catch (ReplicaCatalogException e) {
                            log.error(e.getMessage(), e);
                        }
                    } else if (pi.getType().equals(DataType.URI_COLLECTION) && pi.getValue().contains("airavata-dp://")) {
                        try {
                            String[] uriList = pi.getValue().split(",");
                            final ArrayList<String> filePathList = new ArrayList<>();
                            for (String uri : uriList) {
                                if (uri.startsWith("airavata-dp://")) {
                                    DataProductModel dataProductModel = replicaCatalog.getDataProduct(uri);
                                    Optional<DataReplicaLocationModel> rpLocation = dataProductModel.getReplicaLocations().stream().filter(rpModel -> rpModel.getReplicaLocationCategory().equals(ReplicaLocationCategory.GATEWAY_DATA_STORE)).findFirst();
                                    if (rpLocation.isPresent()) {
                                        filePathList.add(rpLocation.get().getFilePath());
                                    } else {
                                        log.error("Could not find a replica for the URI " + pi.getValue());
                                    }
                                } else {
                                    // uri is in file path format
                                    filePathList.add(uri);
                                }
                            }
                            pi.setValue(StringUtils.join(filePathList, ','));
                        } catch (ReplicaCatalogException e) {
                            log.error(e.getMessage(), e);
                        }
                    }
                });
                String taskDag = orchestrator.createAndSaveTasks(gatewayId, processModel, experiment.getUserConfigurationData().isAiravataAutoSchedule());
                processModel.setTaskDag(taskDag);
                experimentCatalog.update(ExperimentCatalogModelType.PROCESS, processModel, processModel.getProcessId());
            }
            if (!validateProcess(experimentId, processes)) {
                log.error("Validating process fails for given experiment Id : {}", experimentId);
                return false;
            }
            log.debug(experimentId, "Launching single application experiment {}.", experimentId);
            ExperimentStatus status = new ExperimentStatus(ExperimentState.LAUNCHED);
            status.setReason("submitted all processes");
            status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
            OrchestratorUtils.updageAndPublishExperimentStatus(experimentId, status, publisher, gatewayId);
            log.info("expId: {}, Launched experiment ", experimentId);
            OrchestratorServerThreadPoolExecutor.getCachedThreadPool().execute(MDCUtil.wrapWithMDC(new SingleAppExperimentRunner(experimentId, token, gatewayId)));
        } else if (executionType == ExperimentType.WORKFLOW) {
            // its a workflow execution experiment
            log.debug(experimentId, "Launching workflow experiment {}.", experimentId);
            launchWorkflowExperiment(experimentId, token, gatewayId);
        } else {
            log.error(experimentId, "Couldn't identify experiment type, experiment {} is neither single application nor workflow.", experimentId);
            throw new TException("Experiment '" + experimentId + "' launch failed. Unable to figureout execution type for application " + experiment.getExecutionId());
        }
    } catch (LaunchValidationException launchValidationException) {
        ExperimentStatus status = new ExperimentStatus(ExperimentState.FAILED);
        status.setReason("Validation failed: " + launchValidationException.getErrorMessage());
        status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
        OrchestratorUtils.updageAndPublishExperimentStatus(experimentId, status, publisher, gatewayId);
        throw new TException("Experiment '" + experimentId + "' launch failed. Experiment failed to validate: " + launchValidationException.getErrorMessage(), launchValidationException);
    } catch (Exception e) {
        throw new TException("Experiment '" + experimentId + "' launch failed. Unable to figureout execution type for application " + experiment.getExecutionId(), e);
    }
    return true;
}

Example 23 with ExperimentModel

use of org.apache.airavata.model.experiment.ExperimentModel in project airavata by apache.

the class OrchestratorClientSample method storeExperimentDetail.

/*

    public static void main(String[] args) {
        try {
            AiravataUtils.setExecutionAsClient();
            sysUser = ClientSettings.getSetting(DEFAULT_USER);
            sysUserPwd = ClientSettings.getSetting(DEFAULT_USER_PASSWORD);
            gateway = ClientSettings.getSetting(DEFAULT_GATEWAY);
            orchestratorClient = OrchestratorClientFactory.createOrchestratorClient("localhost", 8940);
            registry = RegistryFactory.getRegistry(gateway, sysUser, sysUserPwd);
            documentCreator = new DocumentCreator(getAiravataAPI());
            documentCreator.createLocalHostDocs();
            documentCreator.createGramDocs();
            documentCreator.createPBSDocsForOGCE();
            storeExperimentDetail();
        } catch (ApplicationSettingsException e) {
            e.printStackTrace();
        } catch (RegistryException e) {
            e.printStackTrace();
        }

    }

    private static AiravataAPI getAiravataAPI() {
        AiravataAPI airavataAPI = null;
            try {
                airavataAPI = AiravataAPIFactory.getAPI(gateway, sysUser);
            } catch (AiravataAPIInvocationException e) {
                e.printStackTrace();
            }
        return airavataAPI;
    }
*/
public static void storeExperimentDetail() {
    for (int i = 0; i < NUM_CONCURRENT_REQUESTS; i++) {
        Thread thread = new Thread() {

            public void run() {
                List<InputDataObjectType> exInputs = new ArrayList<InputDataObjectType>();
                InputDataObjectType input = new InputDataObjectType();
                input.setName("echo_input");
                input.setType(DataType.STRING);
                input.setValue("echo_output=Hello World");
                exInputs.add(input);
                List<OutputDataObjectType> exOut = new ArrayList<OutputDataObjectType>();
                OutputDataObjectType output = new OutputDataObjectType();
                output.setName("echo_output");
                output.setType(DataType.STRING);
                output.setValue("");
                exOut.add(output);
                ExperimentModel simpleExperiment = ExperimentModelUtil.createSimpleExperiment(DEFAULT_GATEWAY, "default", "admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs);
                simpleExperiment.setExperimentOutputs(exOut);
                ComputationalResourceSchedulingModel scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 1, 1, 1, "normal", 0, 0);
                scheduling.setResourceHostId("gsissh-trestles");
                UserConfigurationDataModel userConfigurationDataModel = new UserConfigurationDataModel();
                userConfigurationDataModel.setComputationalResourceScheduling(scheduling);
                simpleExperiment.setUserConfigurationData(userConfigurationDataModel);
                String expId = null;
                try {
                // expId = (String) registry.add(ParentDataType.EXPERIMENT, simpleExperiment);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                try {
                    orchestratorClient.launchExperiment(expId, "airavataToken");
                } catch (TException e) {
                    e.printStackTrace();
                }
            }
        };
        thread.start();
        try {
            thread.join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

Example 24 with ExperimentModel

use of org.apache.airavata.model.experiment.ExperimentModel in project airavata by apache.

the class ExperimentRegistry method getExperimentList.

public List<ExperimentModel> getExperimentList(String fieldName, Object value) throws RegistryException {
    List<ExperimentModel> experiments = new ArrayList<ExperimentModel>();
    try {
        if (fieldName.equals(Constants.FieldConstants.ExperimentConstants.USER_NAME)) {
            WorkerResource resource = (WorkerResource) gatewayResource.create(ResourceType.GATEWAY_WORKER);
            resource.setUser((String) value);
            List<ExperimentResource> resources = resource.getExperiments();
            for (ExperimentResource experimentResource : resources) {
                ExperimentModel experiment = ThriftDataModelConversion.getExperiment(experimentResource);
                experiments.add(experiment);
            }
            return experiments;
        } else if (fieldName.equals(Constants.FieldConstants.ExperimentConstants.PROJECT_ID)) {
            ProjectResource project = workerResource.getProject((String) value);
            List<ExperimentResource> resources = project.getExperiments();
            for (ExperimentResource resource : resources) {
                ExperimentModel experiment = ThriftDataModelConversion.getExperiment(resource);
                experiments.add(experiment);
            }
            return experiments;
        } else if (fieldName.equals(Constants.FieldConstants.ExperimentConstants.GATEWAY_ID)) {
            List<ExperimentResource> resources = gatewayResource.getExperiments();
            for (ExperimentResource resource : resources) {
                ExperimentModel experiment = ThriftDataModelConversion.getExperiment(resource);
                experiments.add(experiment);
            }
            return experiments;
        } else {
            logger.error("Unsupported field name to retrieve experiment list...");
        }
    } catch (Exception e) {
        logger.error("Error while getting experiment list...", e);
        throw new RegistryException(e);
    }
    return experiments;
}

Example 25 with ExperimentModel

use of org.apache.airavata.model.experiment.ExperimentModel in project airavata by apache.

the class ThriftDataModelConversion method getExperiment.

public static ExperimentModel getExperiment(ExperimentResource experimentResource) throws RegistryException {
    if (experimentResource != null) {
        ExperimentModel experiment = new ExperimentModel();
        experiment.setProjectId(experimentResource.getProjectId());
        experiment.setExperimentId(experimentResource.getExperimentId());
        experiment.setGatewayId(experimentResource.getGatewayId());
        experiment.setCreationTime(experimentResource.getCreationTime().getTime());
        experiment.setUserName(experimentResource.getUserName());
        experiment.setExperimentName(experimentResource.getExperimentName());
        experiment.setExecutionId(experimentResource.getExecutionId());
        experiment.setDescription(experimentResource.getDescription());
        experiment.setEnableEmailNotification(experimentResource.getEnableEmailNotification());
        experiment.setGatewayExecutionId(experimentResource.getGatewayExecutionId());
        experiment.setGatewayInstanceId(experimentResource.getGatewayInstanceId());
        if (experiment.isEnableEmailNotification()) {
            String notificationEmails = experimentResource.getEmailAddresses();
            experiment.setEmailAddresses(getEmailAddresses(notificationEmails.split(",")));
        }
        List<ExperimentInputResource> experimentInputs = experimentResource.getExperimentInputs();
        experiment.setExperimentInputs(getExpInputs(experimentInputs));
        List<ExperimentOutputResource> experimentOutputs = experimentResource.getExperimentOutputs();
        experiment.setExperimentOutputs(getExpOutputs(experimentOutputs));
        ExperimentStatusResource experimentStatus = experimentResource.getExperimentStatus();
        if (experimentStatus != null) {
            List<ExperimentStatus> experimentStatuses = new ArrayList<>();
            experimentStatuses.add(getExperimentStatus(experimentStatus));
            experiment.setExperimentStatus(experimentStatuses);
        }
        List<ExperimentErrorResource> errorDetails = experimentResource.getExperimentErrors();
        if (errorDetails != null && !errorDetails.isEmpty()) {
            experiment.setErrors(getExperimentErrorList(errorDetails));
        }
        UserConfigurationDataResource userConfigurationDataResource = experimentResource.getUserConfigurationDataResource();
        if (userConfigurationDataResource != null) {
            experiment.setUserConfigurationData(getUserConfigData(userConfigurationDataResource));
        }
        return experiment;
    }
    return null;
}