Search in sources :

Example 1 with DataProductModel

use of org.apache.airavata.model.data.replica.DataProductModel in project airavata by apache.

the class AiravataServerHandler method getDataProduct.

@Override
@SecurityCheck
public DataProductModel getDataProduct(AuthzToken authzToken, String productUri) throws InvalidRequestException, AiravataClientException, AiravataSystemException, AuthorizationException, TException {
    RegistryService.Client regClient = registryClientPool.getResource();
    try {
        DataProductModel result = regClient.getDataProduct(productUri);
        registryClientPool.returnResource(regClient);
        return result;
    } catch (Exception e) {
        String msg = "Error in retreiving the data product " + productUri + ".";
        logger.error(msg, e);
        AiravataSystemException exception = new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
        exception.setMessage(msg + " More info : " + e.getMessage());
        registryClientPool.returnBrokenResource(regClient);
        throw exception;
    }
}
Also used : RegistryService(org.apache.airavata.registry.api.RegistryService) SharingRegistryService(org.apache.airavata.sharing.registry.service.cpi.SharingRegistryService) RegistryServiceException(org.apache.airavata.registry.api.exception.RegistryServiceException) CredentialStoreException(org.apache.airavata.credential.store.exception.CredentialStoreException) AiravataException(org.apache.airavata.common.exception.AiravataException) TException(org.apache.thrift.TException) ApplicationSettingsException(org.apache.airavata.common.exception.ApplicationSettingsException) DataProductModel(org.apache.airavata.model.data.replica.DataProductModel) SecurityCheck(org.apache.airavata.service.security.interceptor.SecurityCheck)

Example 2 with DataProductModel

use of org.apache.airavata.model.data.replica.DataProductModel in project airavata by apache.

the class RegistryServerHandler method getDataProduct.

@Override
public DataProductModel getDataProduct(String productUri) throws RegistryServiceException, TException {
    try {
        dataCatalog = RegistryFactory.getReplicaCatalog();
        DataProductModel dataProductModel = dataCatalog.getDataProduct(productUri);
        return dataProductModel;
    } catch (RegistryException e) {
        String msg = "Error in retreiving the data product " + productUri + ".";
        logger.error(msg, e);
        RegistryServiceException exception = new RegistryServiceException();
        exception.setMessage(msg + " More info : " + e.getMessage());
        throw exception;
    }
}
Also used : RegistryServiceException(org.apache.airavata.registry.api.exception.RegistryServiceException) DataProductModel(org.apache.airavata.model.data.replica.DataProductModel)

Example 3 with DataProductModel

use of org.apache.airavata.model.data.replica.DataProductModel in project airavata by apache.

the class RegistryServerHandler method getParentDataProduct.

@Override
public DataProductModel getParentDataProduct(String productUri) throws RegistryServiceException, TException {
    try {
        dataCatalog = RegistryFactory.getReplicaCatalog();
        DataProductModel dataProductModel = dataCatalog.getParentDataProduct(productUri);
        return dataProductModel;
    } catch (RegistryException e) {
        String msg = "Error in retreiving the parent data product for " + productUri + ".";
        logger.error(msg, e);
        RegistryServiceException exception = new RegistryServiceException();
        exception.setMessage(msg + " More info : " + e.getMessage());
        throw exception;
    }
}
Also used : RegistryServiceException(org.apache.airavata.registry.api.exception.RegistryServiceException) DataProductModel(org.apache.airavata.model.data.replica.DataProductModel)

Example 4 with DataProductModel

use of org.apache.airavata.model.data.replica.DataProductModel in project airavata by apache.

the class OrchestratorServerHandler method launchExperiment.

/**
 * * After creating the experiment Data user have the * experimentID as the
 * handler to the experiment, during the launchProcess * We just have to
 * give the experimentID * * @param experimentID * @return sucess/failure *
 * *
 *
 * @param experimentId
 */
public boolean launchExperiment(String experimentId, String gatewayId) throws TException {
    ExperimentModel experiment = null;
    try {
        String experimentNodePath = GFacUtils.getExperimentNodePath(experimentId);
        ZKPaths.mkdirs(curatorClient.getZookeeperClient().getZooKeeper(), experimentNodePath);
        String experimentCancelNode = ZKPaths.makePath(experimentNodePath, ZkConstants.ZOOKEEPER_CANCEL_LISTENER_NODE);
        ZKPaths.mkdirs(curatorClient.getZookeeperClient().getZooKeeper(), experimentCancelNode);
        experiment = (ExperimentModel) experimentCatalog.get(ExperimentCatalogModelType.EXPERIMENT, experimentId);
        if (experiment == null) {
            log.error("Error retrieving the Experiment by the given experimentID: {} ", experimentId);
            return false;
        }
        ComputeResourcePreference computeResourcePreference = appCatalog.getGatewayProfile().getComputeResourcePreference(gatewayId, experiment.getUserConfigurationData().getComputationalResourceScheduling().getResourceHostId());
        String token = computeResourcePreference.getResourceSpecificCredentialStoreToken();
        if (token == null || token.isEmpty()) {
            // try with gateway profile level token
            GatewayResourceProfile gatewayProfile = appCatalog.getGatewayProfile().getGatewayProfile(gatewayId);
            token = gatewayProfile.getCredentialStoreToken();
        }
        // still the token is empty, then we fail the experiment
        if (token == null || token.isEmpty()) {
            log.error("You have not configured credential store token at gateway profile or compute resource preference." + " Please provide the correct token at gateway profile or compute resource preference.");
            return false;
        }
        ExperimentType executionType = experiment.getExperimentType();
        if (executionType == ExperimentType.SINGLE_APPLICATION) {
            // its an single application execution experiment
            List<ProcessModel> processes = orchestrator.createProcesses(experimentId, gatewayId);
            for (ProcessModel processModel : processes) {
                // FIXME Resolving replica if available. This is a very crude way of resolving input replicas. A full featured
                // FIXME replica resolving logic should come here
                ReplicaCatalog replicaCatalog = RegistryFactory.getReplicaCatalog();
                processModel.getProcessInputs().stream().forEach(pi -> {
                    if (pi.getType().equals(DataType.URI) && pi.getValue().startsWith("airavata-dp://")) {
                        try {
                            DataProductModel dataProductModel = replicaCatalog.getDataProduct(pi.getValue());
                            Optional<DataReplicaLocationModel> rpLocation = dataProductModel.getReplicaLocations().stream().filter(rpModel -> rpModel.getReplicaLocationCategory().equals(ReplicaLocationCategory.GATEWAY_DATA_STORE)).findFirst();
                            if (rpLocation.isPresent()) {
                                pi.setValue(rpLocation.get().getFilePath());
                                pi.setStorageResourceId(rpLocation.get().getStorageResourceId());
                            } else {
                                log.error("Could not find a replica for the URI " + pi.getValue());
                            }
                        } catch (ReplicaCatalogException e) {
                            log.error(e.getMessage(), e);
                        }
                    } else if (pi.getType().equals(DataType.URI_COLLECTION) && pi.getValue().contains("airavata-dp://")) {
                        try {
                            String[] uriList = pi.getValue().split(",");
                            final ArrayList<String> filePathList = new ArrayList<>();
                            for (String uri : uriList) {
                                if (uri.startsWith("airavata-dp://")) {
                                    DataProductModel dataProductModel = replicaCatalog.getDataProduct(uri);
                                    Optional<DataReplicaLocationModel> rpLocation = dataProductModel.getReplicaLocations().stream().filter(rpModel -> rpModel.getReplicaLocationCategory().equals(ReplicaLocationCategory.GATEWAY_DATA_STORE)).findFirst();
                                    if (rpLocation.isPresent()) {
                                        filePathList.add(rpLocation.get().getFilePath());
                                    } else {
                                        log.error("Could not find a replica for the URI " + pi.getValue());
                                    }
                                } else {
                                    // uri is in file path format
                                    filePathList.add(uri);
                                }
                            }
                            pi.setValue(StringUtils.join(filePathList, ','));
                        } catch (ReplicaCatalogException e) {
                            log.error(e.getMessage(), e);
                        }
                    }
                });
                String taskDag = orchestrator.createAndSaveTasks(gatewayId, processModel, experiment.getUserConfigurationData().isAiravataAutoSchedule());
                processModel.setTaskDag(taskDag);
                experimentCatalog.update(ExperimentCatalogModelType.PROCESS, processModel, processModel.getProcessId());
            }
            if (!validateProcess(experimentId, processes)) {
                log.error("Validating process fails for given experiment Id : {}", experimentId);
                return false;
            }
            log.debug(experimentId, "Launching single application experiment {}.", experimentId);
            ExperimentStatus status = new ExperimentStatus(ExperimentState.LAUNCHED);
            status.setReason("submitted all processes");
            status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
            OrchestratorUtils.updageAndPublishExperimentStatus(experimentId, status, publisher, gatewayId);
            log.info("expId: {}, Launched experiment ", experimentId);
            OrchestratorServerThreadPoolExecutor.getCachedThreadPool().execute(MDCUtil.wrapWithMDC(new SingleAppExperimentRunner(experimentId, token, gatewayId)));
        } else if (executionType == ExperimentType.WORKFLOW) {
            // its a workflow execution experiment
            log.debug(experimentId, "Launching workflow experiment {}.", experimentId);
            launchWorkflowExperiment(experimentId, token, gatewayId);
        } else {
            log.error(experimentId, "Couldn't identify experiment type, experiment {} is neither single application nor workflow.", experimentId);
            throw new TException("Experiment '" + experimentId + "' launch failed. Unable to figureout execution type for application " + experiment.getExecutionId());
        }
    } catch (LaunchValidationException launchValidationException) {
        ExperimentStatus status = new ExperimentStatus(ExperimentState.FAILED);
        status.setReason("Validation failed: " + launchValidationException.getErrorMessage());
        status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
        OrchestratorUtils.updageAndPublishExperimentStatus(experimentId, status, publisher, gatewayId);
        throw new TException("Experiment '" + experimentId + "' launch failed. Experiment failed to validate: " + launchValidationException.getErrorMessage(), launchValidationException);
    } catch (Exception e) {
        throw new TException("Experiment '" + experimentId + "' launch failed. Unable to figureout execution type for application " + experiment.getExecutionId(), e);
    }
    return true;
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) MDCUtil(org.apache.airavata.common.logging.MDCUtil) LoggerFactory(org.slf4j.LoggerFactory) org.apache.airavata.orchestrator.cpi.orchestrator_cpiConstants(org.apache.airavata.orchestrator.cpi.orchestrator_cpiConstants) Stat(org.apache.zookeeper.data.Stat) DataReplicaLocationModel(org.apache.airavata.model.data.replica.DataReplicaLocationModel) OrchestratorException(org.apache.airavata.orchestrator.core.exception.OrchestratorException) ZKPaths(org.apache.curator.utils.ZKPaths) ServerSettings(org.apache.airavata.common.utils.ServerSettings) TBase(org.apache.thrift.TBase) OrchestratorServerThreadPoolExecutor(org.apache.airavata.orchestrator.util.OrchestratorServerThreadPoolExecutor) AbstractExpCatResource(org.apache.airavata.registry.core.experiment.catalog.resources.AbstractExpCatResource) ZkConstants(org.apache.airavata.common.utils.ZkConstants) ApplicationInterfaceDescription(org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription) OrchestratorService(org.apache.airavata.orchestrator.cpi.OrchestratorService) DataType(org.apache.airavata.model.application.io.DataType) ValidationResults(org.apache.airavata.model.error.ValidationResults) HostScheduler(org.apache.airavata.gfac.core.scheduler.HostScheduler) AppCatAbstractResource(org.apache.airavata.registry.core.app.catalog.resources.AppCatAbstractResource) CuratorFramework(org.apache.curator.framework.CuratorFramework) ErrorModel(org.apache.airavata.model.commons.ErrorModel) RetryPolicy(org.apache.curator.RetryPolicy) AiravataUtils(org.apache.airavata.common.utils.AiravataUtils) CuratorFrameworkFactory(org.apache.curator.framework.CuratorFrameworkFactory) java.util(java.util) SimpleOrchestratorImpl(org.apache.airavata.orchestrator.cpi.impl.SimpleOrchestratorImpl) MDCConstants(org.apache.airavata.common.logging.MDCConstants) ExponentialBackoffRetry(org.apache.curator.retry.ExponentialBackoffRetry) ReplicaLocationCategory(org.apache.airavata.model.data.replica.ReplicaLocationCategory) ExperimentStatus(org.apache.airavata.model.status.ExperimentStatus) AiravataException(org.apache.airavata.common.exception.AiravataException) GFacUtils(org.apache.airavata.gfac.core.GFacUtils) RegistryFactory(org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory) LaunchValidationException(org.apache.airavata.model.error.LaunchValidationException) Logger(org.slf4j.Logger) org.apache.airavata.messaging.core(org.apache.airavata.messaging.core) ExperimentState(org.apache.airavata.model.status.ExperimentState) TException(org.apache.thrift.TException) ExperimentType(org.apache.airavata.model.experiment.ExperimentType) DataProductModel(org.apache.airavata.model.data.replica.DataProductModel) ThriftUtils(org.apache.airavata.common.utils.ThriftUtils) org.apache.airavata.registry.cpi(org.apache.airavata.registry.cpi) ComputeResourcePreference(org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference) GatewayResourceProfile(org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile) ExperimentModel(org.apache.airavata.model.experiment.ExperimentModel) ApplicationDeploymentDescription(org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription) MDC(org.slf4j.MDC) org.apache.airavata.model.messaging.event(org.apache.airavata.model.messaging.event) OrchestratorUtils(org.apache.airavata.orchestrator.util.OrchestratorUtils) ComputeResourceDescription(org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription) ProcessModel(org.apache.airavata.model.process.ProcessModel) ApplicationSettingsException(org.apache.airavata.common.exception.ApplicationSettingsException) TException(org.apache.thrift.TException) ComputeResourcePreference(org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference) ProcessModel(org.apache.airavata.model.process.ProcessModel) GatewayResourceProfile(org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile) ExperimentModel(org.apache.airavata.model.experiment.ExperimentModel) LaunchValidationException(org.apache.airavata.model.error.LaunchValidationException) DataReplicaLocationModel(org.apache.airavata.model.data.replica.DataReplicaLocationModel) OrchestratorException(org.apache.airavata.orchestrator.core.exception.OrchestratorException) AiravataException(org.apache.airavata.common.exception.AiravataException) LaunchValidationException(org.apache.airavata.model.error.LaunchValidationException) TException(org.apache.thrift.TException) ApplicationSettingsException(org.apache.airavata.common.exception.ApplicationSettingsException) DataProductModel(org.apache.airavata.model.data.replica.DataProductModel) ExperimentStatus(org.apache.airavata.model.status.ExperimentStatus) ExperimentType(org.apache.airavata.model.experiment.ExperimentType)

Example 5 with DataProductModel

use of org.apache.airavata.model.data.replica.DataProductModel in project airavata by apache.

the class ReplicaCatalogImpl method searchDataProductsByName.

@Override
public List<DataProductModel> searchDataProductsByName(String gatewayId, String userId, String productName, int limit, int offset) throws ReplicaCatalogException {
    EntityManager em = null;
    try {
        String query = "SELECT dp FROM DataProduct dp " + "WHERE dp.gatewayId = '" + gatewayId + "' AND dp.ownerName='" + userId + "' AND dp.productName LIKE '%" + productName + "%' ORDER BY dp.creationTime DESC";
        em = ReplicaCatalogJPAUtils.getEntityManager();
        em.getTransaction().begin();
        Query q;
        // pagination
        if (offset >= 0 && limit >= 0) {
            q = em.createQuery(query).setFirstResult(offset).setMaxResults(limit);
        } else {
            q = em.createQuery(query);
        }
        ArrayList<DataProductModel> returnModels = new ArrayList<>();
        List resultList = q.getResultList();
        for (Object o : resultList) {
            DataProduct dataProduct = (DataProduct) o;
            returnModels.add(ThriftDataModelConversion.getDataProductModel(dataProduct));
        }
        em.getTransaction().commit();
        em.close();
        return returnModels;
    } catch (Exception e) {
        e.printStackTrace();
        logger.error(e.getMessage(), e);
        throw new ReplicaCatalogException(e);
    } finally {
        if (em != null && em.isOpen()) {
            if (em.getTransaction().isActive()) {
                em.getTransaction().rollback();
            }
            em.close();
        }
    }
}
Also used : EntityManager(javax.persistence.EntityManager) Query(javax.persistence.Query) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) DataProduct(org.apache.airavata.registry.core.replica.catalog.model.DataProduct) ReplicaCatalogException(org.apache.airavata.registry.cpi.ReplicaCatalogException) ReplicaCatalogException(org.apache.airavata.registry.cpi.ReplicaCatalogException) DataProductModel(org.apache.airavata.model.data.replica.DataProductModel)

Aggregations

DataProductModel (org.apache.airavata.model.data.replica.DataProductModel)9 RegistryServiceException (org.apache.airavata.registry.api.exception.RegistryServiceException)6 AiravataException (org.apache.airavata.common.exception.AiravataException)3 ApplicationSettingsException (org.apache.airavata.common.exception.ApplicationSettingsException)3 TException (org.apache.thrift.TException)3 ArrayList (java.util.ArrayList)2 EntityManager (javax.persistence.EntityManager)2 CredentialStoreException (org.apache.airavata.credential.store.exception.CredentialStoreException)2 RegistryService (org.apache.airavata.registry.api.RegistryService)2 DataProduct (org.apache.airavata.registry.core.replica.catalog.model.DataProduct)2 ReplicaCatalogException (org.apache.airavata.registry.cpi.ReplicaCatalogException)2 java.util (java.util)1 List (java.util.List)1 Query (javax.persistence.Query)1 MDCConstants (org.apache.airavata.common.logging.MDCConstants)1 MDCUtil (org.apache.airavata.common.logging.MDCUtil)1 AiravataUtils (org.apache.airavata.common.utils.AiravataUtils)1 ServerSettings (org.apache.airavata.common.utils.ServerSettings)1 ThriftUtils (org.apache.airavata.common.utils.ThriftUtils)1 ZkConstants (org.apache.airavata.common.utils.ZkConstants)1