Examples with InputDataObjectType org.apache.airavata.model.application.io.InputDataObjectType used on opensource projects

Example 1 with InputDataObjectType

use of org.apache.airavata.model.application.io.InputDataObjectType in project airavata by apache.

the class BESJobSubmissionTask method copyInputFilesToLocal.

private void copyInputFilesToLocal(TaskContext taskContext) throws GFacException {
    ProcessContext pc = taskContext.getParentProcessContext();
    StorageResourceDescription storageResource = pc.getStorageResource();
    if (storageResource != null) {
        hostName = storageResource.getHostName();
    } else {
        throw new GFacException("Storage Resource is null");
    }
    inputPath = pc.getStorageFileSystemRootLocation();
    inputPath = (inputPath.endsWith(File.separator) ? inputPath : inputPath + File.separator);
    String remoteFilePath = null, fileName = null, localFilePath = null;
    URI remoteFileURI = null;
    try {
        authenticationInfo = Factory.getStorageSSHKeyAuthentication(pc);
        ServerInfo serverInfo = pc.getStorageResourceServerInfo();
        Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo);
        List<InputDataObjectType> processInputs = pc.getProcessModel().getProcessInputs();
        for (InputDataObjectType input : processInputs) {
            if (input.getType() == DataType.URI) {
                remoteFileURI = new URI(input.getValue());
                remoteFilePath = remoteFileURI.getPath();
                fileName = remoteFilePath.substring(remoteFilePath.lastIndexOf("/") + 1);
                localFilePath = pc.getInputDir() + File.separator + fileName;
                log.info("SCP remote file :{} -> to local :{}", remoteFilePath, localFilePath);
                SSHUtils.scpFrom(remoteFilePath, localFilePath, sshSession);
                input.setValue("file:/" + localFilePath);
            }
        }
    } catch (IOException | JSchException | SSHApiException | URISyntaxException e) {
        log.error("Error while coping remote file " + remoteFilePath + " to local " + localFilePath, e);
        throw new GFacException("Error while scp input files to local file location", e);
    } catch (CredentialStoreException e) {
        String msg = "Authentication issue, make sure you are passing valid credential token";
        log.error(msg, e);
        throw new GFacException(msg, e);
    }
}

Example 2 with InputDataObjectType

use of org.apache.airavata.model.application.io.InputDataObjectType in project airavata by apache.

the class LocalJobSubmissionTask method buildCommand.

private List<String> buildCommand(ProcessContext processContext) {
    List<String> cmdList = new ArrayList<>();
    cmdList.add(processContext.getApplicationDeploymentDescription().getExecutablePath());
    List<InputDataObjectType> processInputs = processContext.getProcessModel().getProcessInputs();
    // sort the inputs first and then build the command List
    Comparator<InputDataObjectType> inputOrderComparator = new Comparator<InputDataObjectType>() {

        @Override
        public int compare(InputDataObjectType inputDataObjectType, InputDataObjectType t1) {
            return inputDataObjectType.getInputOrder() - t1.getInputOrder();
        }
    };
    Set<InputDataObjectType> sortedInputSet = new TreeSet<InputDataObjectType>(inputOrderComparator);
    for (InputDataObjectType input : processInputs) {
        sortedInputSet.add(input);
    }
    for (InputDataObjectType inputDataObjectType : sortedInputSet) {
        if (inputDataObjectType.getApplicationArgument() != null && !inputDataObjectType.getApplicationArgument().equals("")) {
            cmdList.add(inputDataObjectType.getApplicationArgument());
        }
        if (inputDataObjectType.getValue() != null && !inputDataObjectType.getValue().equals("")) {
            cmdList.add(inputDataObjectType.getValue());
        }
    }
    return cmdList;
}

Example 3 with InputDataObjectType

use of org.apache.airavata.model.application.io.InputDataObjectType in project airavata by apache.

the class SCPDataStageTask method execute.

@Override
public TaskStatus execute(TaskContext taskContext) {
    TaskStatus status = new TaskStatus(TaskState.EXECUTING);
    AuthenticationInfo authenticationInfo = null;
    DataStagingTaskModel subTaskModel = null;
    String localDataDir = null;
    ProcessContext processContext = taskContext.getParentProcessContext();
    ProcessState processState = processContext.getProcessState();
    try {
        subTaskModel = ((DataStagingTaskModel) taskContext.getSubTaskModel());
        if (processState == ProcessState.OUTPUT_DATA_STAGING) {
            OutputDataObjectType processOutput = taskContext.getProcessOutput();
            if (processOutput != null && processOutput.getValue() == null) {
                log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , file name shouldn't be null", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), processOutput.getName());
                status = new TaskStatus(TaskState.FAILED);
                if (processOutput.isIsRequired()) {
                    status.setReason("File name is null, but this output's isRequired bit is not set");
                } else {
                    status.setReason("File name is null");
                }
                return status;
            }
        } else if (processState == ProcessState.INPUT_DATA_STAGING) {
            InputDataObjectType processInput = taskContext.getProcessInput();
            if (processInput != null && processInput.getValue() == null) {
                log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , file name shouldn't be null", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), processInput.getName());
                status = new TaskStatus(TaskState.FAILED);
                if (processInput.isIsRequired()) {
                    status.setReason("File name is null, but this input's isRequired bit is not set");
                } else {
                    status.setReason("File name is null");
                }
                return status;
            }
        } else {
            status.setState(TaskState.FAILED);
            status.setReason("Invalid task invocation, Support " + ProcessState.INPUT_DATA_STAGING.name() + " and " + "" + ProcessState.OUTPUT_DATA_STAGING.name() + " process phases. found " + processState.name());
            return status;
        }
        StorageResourceDescription storageResource = processContext.getStorageResource();
        // StoragePreference storagePreference = taskContext.getParentProcessContext().getStoragePreference();
        String hostName = null;
        if (storageResource != null) {
            hostName = storageResource.getHostName();
        } else {
            throw new GFacException("Storage Resource is null");
        }
        String inputPath = processContext.getStorageFileSystemRootLocation();
        inputPath = (inputPath.endsWith(File.separator) ? inputPath : inputPath + File.separator);
        // use rsync instead of scp if source and destination host and user name is same.
        URI sourceURI = new URI(subTaskModel.getSource());
        String fileName = sourceURI.getPath().substring(sourceURI.getPath().lastIndexOf(File.separator) + 1, sourceURI.getPath().length());
        Session remoteSession = Factory.getSSHSession(Factory.getComputerResourceSSHKeyAuthentication(processContext), processContext.getComputeResourceServerInfo());
        Session storageSession = Factory.getSSHSession(Factory.getStorageSSHKeyAuthentication(processContext), processContext.getStorageResourceServerInfo());
        URI destinationURI = null;
        if (subTaskModel.getDestination().startsWith("dummy")) {
            destinationURI = TaskUtils.getDestinationURI(taskContext, hostName, inputPath, fileName);
            subTaskModel.setDestination(destinationURI.toString());
        } else {
            destinationURI = new URI(subTaskModel.getDestination());
        }
        if (sourceURI.getHost().equalsIgnoreCase(destinationURI.getHost()) && sourceURI.getUserInfo().equalsIgnoreCase(destinationURI.getUserInfo())) {
            localDataCopy(taskContext, sourceURI, destinationURI);
            status.setState(TaskState.COMPLETED);
            status.setReason("Locally copied file using 'cp' command ");
            return status;
        }
        status = new TaskStatus(TaskState.COMPLETED);
        // Wildcard for file name. Has to find the correct name.
        if (fileName.contains("*")) {
            String destParentPath = (new File(destinationURI.getPath())).getParentFile().getPath();
            String sourceParentPath = (new File(sourceURI.getPath())).getParentFile().getPath();
            List<String> fileNames = taskContext.getParentProcessContext().getDataMovementRemoteCluster().getFileNameFromExtension(fileName, sourceParentPath, remoteSession);
            ExperimentCatalog experimentCatalog = processContext.getExperimentCatalog();
            String experimentId = processContext.getExperimentId();
            String processId = processContext.getProcessId();
            OutputDataObjectType processOutput = taskContext.getProcessOutput();
            for (int i = 0; i < fileNames.size(); i++) {
                String temp = fileNames.get(i);
                if (temp != null && temp != "") {
                    fileName = temp;
                }
                if (destParentPath.endsWith(File.separator)) {
                    destinationURI = new URI(destParentPath + fileName);
                } else {
                    destinationURI = new URI(destParentPath + File.separator + fileName);
                }
                // Wildcard support is only enabled for output data staging
                if (processState == ProcessState.OUTPUT_DATA_STAGING) {
                    processOutput.setName(fileName);
                    experimentCatalog.add(ExpCatChildDataType.EXPERIMENT_OUTPUT, Arrays.asList(processOutput), experimentId);
                    experimentCatalog.add(ExpCatChildDataType.PROCESS_OUTPUT, Arrays.asList(processOutput), processId);
                    taskContext.setProcessOutput(processOutput);
                    makeDir(taskContext, destinationURI);
                    // TODO - save updated subtask model with new destination
                    outputDataStaging(taskContext, remoteSession, sourceURI, storageSession, destinationURI);
                    status.setReason("Successfully staged output data");
                }
            }
            if (processState == ProcessState.OUTPUT_DATA_STAGING) {
                status.setReason("Successfully staged output data");
            } else {
                status.setReason("Wildcard support is only enabled for output data staging");
            }
        } else {
            if (processState == ProcessState.INPUT_DATA_STAGING) {
                inputDataStaging(taskContext, storageSession, sourceURI, remoteSession, destinationURI);
                status.setReason("Successfully staged input data");
            } else if (processState == ProcessState.OUTPUT_DATA_STAGING) {
                makeDir(taskContext, destinationURI);
                // TODO - save updated subtask model with new destination
                outputDataStaging(taskContext, remoteSession, sourceURI, storageSession, destinationURI);
                status.setReason("Successfully staged output data");
            }
        }
    } catch (TException e) {
        String msg = "Couldn't create subTask model thrift model";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
        return status;
    } catch (ApplicationSettingsException | FileNotFoundException e) {
        String msg = "Failed while reading credentials";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (URISyntaxException e) {
        String msg = "Source or destination uri is not correct source : " + subTaskModel.getSource() + ", " + "destination : " + subTaskModel.getDestination();
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (CredentialStoreException e) {
        String msg = "Storage authentication issue, could be invalid credential token";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (AiravataException e) {
        String msg = "Error while creating ssh session with client";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (JSchException | IOException e) {
        String msg = "Failed to do scp with client";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (RegistryException | GFacException e) {
        String msg = "Data staging failed";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    }
    return status;
}

Example 4 with InputDataObjectType

use of org.apache.airavata.model.application.io.InputDataObjectType in project airavata by apache.

the class AiravataIT method testAddApplication.

@org.testng.annotations.Test(priority = 4)
public void testAddApplication() {
    try {
        logger.info("testAddApplication() -> Adding test application......");
        applicationProperties = setup.getApplicationRegister().addApplications();
        ;
        Assert.assertNotNull(applicationProperties);
        ApplicationModule applicationModule = setup.getApplicationRegister().getApplicationModule(applicationProperties.getApplicationModuleId());
        Assert.assertNotNull(applicationModule);
        Assert.assertEquals(applicationModule.getAppModuleName(), TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_NAME);
        Assert.assertEquals(applicationModule.getAppModuleVersion(), LOCAL_ECHO_VERSION);
        Assert.assertEquals(applicationModule.getAppModuleDescription(), TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_DESCRIPTION);
        Assert.assertEquals(applicationModule.getAppModuleName(), TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_NAME);
        ApplicationInterfaceDescription applicationInterfaceDescription = setup.getApplicationRegister().getApplicationInterfaceDescription(applicationProperties.getApplicationInterfaceId());
        Assert.assertNotNull(applicationInterfaceDescription);
        Assert.assertEquals(applicationInterfaceDescription.getApplicationName(), TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_NAME);
        Assert.assertEquals(applicationInterfaceDescription.getApplicationDescription(), TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_DESCRIPTION);
        InputDataObjectType input = applicationInterfaceDescription.getApplicationInputs().get(0);
        Assert.assertNotNull(input);
        Assert.assertEquals(input.getName(), INPUT_NAME);
        Assert.assertEquals(input.getValue(), INPUT_VALUE);
        Assert.assertEquals(input.getType(), DataType.STRING);
        Assert.assertEquals(input.getApplicationArgument(), null);
        Assert.assertEquals(input.getInputOrder(), 0);
        Assert.assertEquals(input.isIsRequired(), true);
        Assert.assertEquals(input.isRequiredToAddedToCommandLine(), true);
        Assert.assertEquals(input.isStandardInput(), false);
        Assert.assertEquals(input.getUserFriendlyDescription(), INPUT_DESC);
        Assert.assertEquals(input.getMetaData(), null);
        List<OutputDataObjectType> outputDataObjectTypes = applicationInterfaceDescription.getApplicationOutputs();
        Assert.assertNotNull(outputDataObjectTypes.get(0));
        Assert.assertEquals(outputDataObjectTypes.get(0).getName(), STDERR_NAME);
        Assert.assertEquals(outputDataObjectTypes.get(0).getValue(), STDERR_VALUE);
        Assert.assertEquals(outputDataObjectTypes.get(0).getType(), DataType.URI);
        Assert.assertEquals(outputDataObjectTypes.get(0).isIsRequired(), true);
        Assert.assertEquals(outputDataObjectTypes.get(0).isRequiredToAddedToCommandLine(), true);
        Assert.assertEquals(outputDataObjectTypes.get(0).getApplicationArgument(), null);
        Assert.assertNotNull(outputDataObjectTypes.get(1));
        Assert.assertEquals(outputDataObjectTypes.get(1).getName(), STDOUT_NAME);
        Assert.assertEquals(outputDataObjectTypes.get(1).getValue(), STDOUT_VALUE);
        Assert.assertEquals(outputDataObjectTypes.get(1).getType(), DataType.URI);
        Assert.assertEquals(outputDataObjectTypes.get(1).isIsRequired(), true);
        Assert.assertEquals(outputDataObjectTypes.get(1).isRequiredToAddedToCommandLine(), true);
        Assert.assertEquals(outputDataObjectTypes.get(1).getApplicationArgument(), null);
        ApplicationDeploymentDescription applicationDeploymentDescription = setup.getApplicationRegister().getApplicationDeploymentDescription(applicationProperties.getApplicationDeployId());
        Assert.assertNotNull(applicationDeploymentDescription);
        Assert.assertEquals(applicationDeploymentDescription.getExecutablePath(), TestFrameworkConstants.LOCAL_ECHO_JOB_FILE_PATH);
        Assert.assertEquals(applicationDeploymentDescription.getParallelism(), ApplicationParallelismType.SERIAL);
        Assert.assertEquals(applicationDeploymentDescription.getAppDeploymentDescription(), TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_DESCRIPTION);
        Assert.assertEquals(applicationDeploymentDescription.getModuleLoadCmds(), null);
        Assert.assertEquals(applicationDeploymentDescription.getPostJobCommands(), null);
        Assert.assertEquals(applicationDeploymentDescription.getPreJobCommands(), null);
        logger.info("testAddApplication() -> Adding test application." + applicationProperties.toString());
    } catch (Exception e) {
        logger.error("Error occured while testAddApplication", e);
        Assert.fail();
    }
}

Example 5 with InputDataObjectType

use of org.apache.airavata.model.application.io.InputDataObjectType in project airavata by apache.

the class ApplicationRegister method registerLocalEchoInterface.

private String registerLocalEchoInterface(Gateway gateway, String moduleId) throws org.apache.thrift.TException {
    List<String> appModules = new ArrayList<String>();
    appModules.add(moduleId);
    InputDataObjectType input1 = createAppInput(INPUT_NAME, INPUT_VALUE, DataType.STRING, null, 0, true, true, false, INPUT_DESC, null);
    List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>(1);
    applicationInputs.add(input1);
    OutputDataObjectType output1 = createAppOutput(STDOUT_NAME, STDOUT_VALUE, DataType.URI, true, true, null);
    OutputDataObjectType output2 = createAppOutput(STDERR_NAME, STDERR_VALUE, DataType.URI, true, true, null);
    List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>(2);
    applicationOutputs.add(output1);
    applicationOutputs.add(output2);
    String localEchoInterfaceId = airavata.registerApplicationInterface(authzToken, gateway.getGatewayId(), createApplicationInterfaceDescription(TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_NAME, TestFrameworkConstants.AppcatalogConstants.LOCAL_ECHO_DESCRIPTION, appModules, applicationInputs, applicationOutputs));
    System.out.println("Echo Local Application Interface Id " + localEchoInterfaceId);
    return localEchoInterfaceId;
}