Examples with DataMovementProtocol org.apache.airavata.model.data.movement.DataMovementProtocol used on opensource projects

Example 1 with DataMovementProtocol

use of org.apache.airavata.model.data.movement.DataMovementProtocol in project airavata by apache.

the class Factory method getDataMovementRemoteCluster.

public static RemoteCluster getDataMovementRemoteCluster(ProcessContext processContext) throws GFacException, AiravataException, CredentialStoreException {
    String storageResourceId = processContext.getStorageResourceId();
    DataMovementProtocol dataMovementProtocol = processContext.getDataMovementProtocol();
    String key = new StringBuilder(processContext.getComputeResourceLoginUserName()).append(':').append(dataMovementProtocol.name()).append(':').append(storageResourceId).append(":").append(processContext.getStorageResourceCredentialToken()).toString();
    RemoteCluster remoteCluster = remoteClusterMap.get(key);
    if (remoteCluster == null) {
        JobManagerConfiguration jobManagerConfiguration = getJobManagerConfiguration(processContext.getResourceJobManager());
        if (dataMovementProtocol == DataMovementProtocol.LOCAL) {
            remoteCluster = new LocalRemoteCluster(processContext.getStorageResourceServerInfo(), jobManagerConfiguration, null);
        } else if (dataMovementProtocol == DataMovementProtocol.SCP) {
            remoteCluster = new HPCRemoteCluster(processContext.getStorageResourceServerInfo(), jobManagerConfiguration, Factory.getStorageSSHKeyAuthentication(processContext));
        } else {
            throw new GFacException("No remote cluster implementation map to job data movement protocol " + dataMovementProtocol.name());
        }
        remoteClusterMap.put(key, remoteCluster);
    } else {
        AuthenticationInfo authentication = remoteCluster.getAuthentication();
        if (authentication instanceof SSHKeyAuthentication) {
            SSHKeyAuthentication sshKeyAuthentication = (SSHKeyAuthentication) authentication;
            if (!sshKeyAuthentication.getUserName().equals(processContext.getStorageResourceLoginUserName())) {
                JobManagerConfiguration jobManagerConfiguration = getJobManagerConfiguration(processContext.getResourceJobManager());
                dataMovementProtocol = processContext.getDataMovementProtocol();
                if (dataMovementProtocol == DataMovementProtocol.SCP) {
                    remoteCluster = new HPCRemoteCluster(processContext.getStorageResourceServerInfo(), jobManagerConfiguration, Factory.getStorageSSHKeyAuthentication(processContext));
                }
            }
        }
    }
    return remoteCluster;
}

Example 2 with DataMovementProtocol

use of org.apache.airavata.model.data.movement.DataMovementProtocol in project airavata by apache.

the class SimpleOrchestratorImpl method getInputDataStagingTask.

private TaskModel getInputDataStagingTask(ProcessModel processModel, InputDataObjectType processInput, String gatewayId) throws RegistryException, TException, AppCatalogException, TaskException, AiravataException {
    // create new task model for this task
    TaskModel taskModel = new TaskModel();
    taskModel.setParentProcessId(processModel.getProcessId());
    taskModel.setCreationTime(AiravataUtils.getCurrentTimestamp().getTime());
    taskModel.setLastUpdateTime(taskModel.getCreationTime());
    TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
    taskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
    taskModel.setTaskStatuses(Arrays.asList(taskStatus));
    taskModel.setTaskType(TaskTypes.DATA_STAGING);
    // create data staging sub task model
    DataStagingTaskModel submodel = new DataStagingTaskModel();
    ComputeResourceDescription computeResource = orchestratorContext.getRegistry().getAppCatalog().getComputeResource().getComputeResource(processModel.getComputeResourceId());
    String workingDir = OrchestratorUtils.getScratchLocation(orchestratorContext, processModel, gatewayId) + File.separator + processModel.getProcessId() + File.separator;
    URI destination = null;
    try {
        DataMovementProtocol dataMovementProtocol = OrchestratorUtils.getPreferredDataMovementProtocol(orchestratorContext, processModel, gatewayId);
        String loginUserName = OrchestratorUtils.getLoginUserName(orchestratorContext, processModel, gatewayId);
        destination = new URI(dataMovementProtocol.name(), loginUserName, computeResource.getHostName(), OrchestratorUtils.getDataMovementPort(orchestratorContext, processModel, gatewayId), workingDir, null, null);
    } catch (URISyntaxException e) {
        throw new TaskException("Error while constructing destination file URI");
    }
    submodel.setType(DataStageType.INPUT);
    submodel.setSource(processInput.getValue());
    submodel.setProcessInput(processInput);
    submodel.setDestination(destination.toString());
    taskModel.setSubTaskModel(ThriftUtils.serializeThriftObject(submodel));
    return taskModel;
}

Example 3 with DataMovementProtocol

use of org.apache.airavata.model.data.movement.DataMovementProtocol in project airavata by apache.

the class SimpleOrchestratorImpl method getOutputDataStagingTask.

private TaskModel getOutputDataStagingTask(ProcessModel processModel, OutputDataObjectType processOutput, String gatewayId) throws RegistryException, TException, AiravataException {
    try {
        // create new task model for this task
        TaskModel taskModel = new TaskModel();
        taskModel.setParentProcessId(processModel.getProcessId());
        taskModel.setCreationTime(AiravataUtils.getCurrentTimestamp().getTime());
        taskModel.setLastUpdateTime(taskModel.getCreationTime());
        TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
        taskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
        taskModel.setTaskStatuses(Arrays.asList(taskStatus));
        taskModel.setTaskType(TaskTypes.DATA_STAGING);
        ComputeResourceDescription computeResource = orchestratorContext.getRegistry().getAppCatalog().getComputeResource().getComputeResource(processModel.getComputeResourceId());
        String workingDir = OrchestratorUtils.getScratchLocation(orchestratorContext, processModel, gatewayId) + File.separator + processModel.getProcessId() + File.separator;
        DataStagingTaskModel submodel = new DataStagingTaskModel();
        DataMovementProtocol dataMovementProtocol = OrchestratorUtils.getPreferredDataMovementProtocol(orchestratorContext, processModel, gatewayId);
        URI source = null;
        try {
            String loginUserName = OrchestratorUtils.getLoginUserName(orchestratorContext, processModel, gatewayId);
            if (processOutput != null) {
                submodel.setType(DataStageType.OUPUT);
                submodel.setProcessOutput(processOutput);
                source = new URI(dataMovementProtocol.name(), loginUserName, computeResource.getHostName(), OrchestratorUtils.getDataMovementPort(orchestratorContext, processModel, gatewayId), workingDir + processOutput.getValue(), null, null);
            } else {
                // archive
                submodel.setType(DataStageType.ARCHIVE_OUTPUT);
                source = new URI(dataMovementProtocol.name(), loginUserName, computeResource.getHostName(), OrchestratorUtils.getDataMovementPort(orchestratorContext, processModel, gatewayId), workingDir, null, null);
            }
        } catch (URISyntaxException e) {
            throw new TaskException("Error while constructing source file URI");
        }
        // We don't know destination location at this time, data staging task will set this.
        // because destination is required field we set dummy destination
        submodel.setSource(source.toString());
        // We don't know destination location at this time, data staging task will set this.
        // because destination is required field we set dummy destination
        submodel.setDestination("dummy://temp/file/location");
        taskModel.setSubTaskModel(ThriftUtils.serializeThriftObject(submodel));
        return taskModel;
    } catch (AppCatalogException | TaskException e) {
        throw new RegistryException("Error occurred while retrieving data movement from app catalog", e);
    }
}

Example 4 with DataMovementProtocol

use of org.apache.airavata.model.data.movement.DataMovementProtocol in project airavata by apache.

the class OrchestratorUtils method getDataMovementPort.

public static int getDataMovementPort(OrchestratorContext context, ProcessModel processModel, String gatewayId) throws RegistryException {
    try {
        DataMovementProtocol protocol = getPreferredDataMovementProtocol(context, processModel, gatewayId);
        DataMovementInterface dataMovementInterface = getPrefferredDataMovementInterface(context, processModel, gatewayId);
        if (protocol == DataMovementProtocol.SCP) {
            SCPDataMovement scpDataMovement = getSCPDataMovement(context, dataMovementInterface.getDataMovementInterfaceId());
            if (scpDataMovement != null) {
                return scpDataMovement.getSshPort();
            }
        }
    } catch (RegistryException e) {
        logger.error("Error occurred while retrieving security protocol", e);
    }
    return 0;
}

Example 5 with DataMovementProtocol

use of org.apache.airavata.model.data.movement.DataMovementProtocol in project airavata by apache.

the class OrchestratorUtils method getPrefferredDataMovementInterface.

public static DataMovementInterface getPrefferredDataMovementInterface(OrchestratorContext context, ProcessModel processModel, String gatewayId) throws RegistryException {
    try {
        String resourceHostId = processModel.getComputeResourceId();
        ComputeResourcePreference resourcePreference = getComputeResourcePreference(context, processModel, gatewayId);
        DataMovementProtocol preferredDataMovementProtocol = resourcePreference.getPreferredDataMovementProtocol();
        ComputeResourceDescription resourceDescription = context.getRegistry().getAppCatalog().getComputeResource().getComputeResource(resourceHostId);
        List<DataMovementInterface> dataMovementInterfaces = resourceDescription.getDataMovementInterfaces();
        if (dataMovementInterfaces != null && !dataMovementInterfaces.isEmpty()) {
            for (DataMovementInterface dataMovementInterface : dataMovementInterfaces) {
                if (preferredDataMovementProtocol != null) {
                    if (preferredDataMovementProtocol.toString().equals(dataMovementInterface.getDataMovementProtocol().toString())) {
                        return dataMovementInterface;
                    }
                }
            }
        } else {
            throw new RegistryException("Compute resource should have at least one data movement interface defined...");
        }
    } catch (AppCatalogException e) {
        throw new RegistryException("Error occurred while retrieving data from app catalog", e);
    }
    return null;
}