Examples with GFacException org.apache.airavata.gfac.core.GFacException used on opensource projects

Example 16 with GFacException

use of org.apache.airavata.gfac.core.GFacException in project airavata by apache.

the class BESJobSubmissionTask method copyInputFilesToLocal.

private void copyInputFilesToLocal(TaskContext taskContext) throws GFacException {
    ProcessContext pc = taskContext.getParentProcessContext();
    StorageResourceDescription storageResource = pc.getStorageResource();
    if (storageResource != null) {
        hostName = storageResource.getHostName();
    } else {
        throw new GFacException("Storage Resource is null");
    }
    inputPath = pc.getStorageFileSystemRootLocation();
    inputPath = (inputPath.endsWith(File.separator) ? inputPath : inputPath + File.separator);
    String remoteFilePath = null, fileName = null, localFilePath = null;
    URI remoteFileURI = null;
    try {
        authenticationInfo = Factory.getStorageSSHKeyAuthentication(pc);
        ServerInfo serverInfo = pc.getStorageResourceServerInfo();
        Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo);
        List<InputDataObjectType> processInputs = pc.getProcessModel().getProcessInputs();
        for (InputDataObjectType input : processInputs) {
            if (input.getType() == DataType.URI) {
                remoteFileURI = new URI(input.getValue());
                remoteFilePath = remoteFileURI.getPath();
                fileName = remoteFilePath.substring(remoteFilePath.lastIndexOf("/") + 1);
                localFilePath = pc.getInputDir() + File.separator + fileName;
                log.info("SCP remote file :{} -> to local :{}", remoteFilePath, localFilePath);
                SSHUtils.scpFrom(remoteFilePath, localFilePath, sshSession);
                input.setValue("file:/" + localFilePath);
            }
        }
    } catch (IOException | JSchException | SSHApiException | URISyntaxException e) {
        log.error("Error while coping remote file " + remoteFilePath + " to local " + localFilePath, e);
        throw new GFacException("Error while scp input files to local file location", e);
    } catch (CredentialStoreException e) {
        String msg = "Authentication issue, make sure you are passing valid credential token";
        log.error(msg, e);
        throw new GFacException(msg, e);
    }
}

Example 17 with GFacException

use of org.apache.airavata.gfac.core.GFacException in project airavata by apache.

the class EnvironmentSetupTask method execute.

@Override
public TaskStatus execute(TaskContext taskContext) {
    TaskStatus status = new TaskStatus(TaskState.COMPLETED);
    try {
        RemoteCluster remoteCluster = taskContext.getParentProcessContext().getJobSubmissionRemoteCluster();
        remoteCluster.makeDirectory(taskContext.getParentProcessContext().getWorkingDir());
        status.setReason("Successfully created environment");
    } catch (GFacException e) {
        String msg = "Error while environment setup";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    }
    return status;
}

Example 18 with GFacException

use of org.apache.airavata.gfac.core.GFacException in project airavata by apache.

the class SCPDataStageTask method execute.

@Override
public TaskStatus execute(TaskContext taskContext) {
    TaskStatus status = new TaskStatus(TaskState.EXECUTING);
    AuthenticationInfo authenticationInfo = null;
    DataStagingTaskModel subTaskModel = null;
    String localDataDir = null;
    ProcessContext processContext = taskContext.getParentProcessContext();
    ProcessState processState = processContext.getProcessState();
    try {
        subTaskModel = ((DataStagingTaskModel) taskContext.getSubTaskModel());
        if (processState == ProcessState.OUTPUT_DATA_STAGING) {
            OutputDataObjectType processOutput = taskContext.getProcessOutput();
            if (processOutput != null && processOutput.getValue() == null) {
                log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , file name shouldn't be null", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), processOutput.getName());
                status = new TaskStatus(TaskState.FAILED);
                if (processOutput.isIsRequired()) {
                    status.setReason("File name is null, but this output's isRequired bit is not set");
                } else {
                    status.setReason("File name is null");
                }
                return status;
            }
        } else if (processState == ProcessState.INPUT_DATA_STAGING) {
            InputDataObjectType processInput = taskContext.getProcessInput();
            if (processInput != null && processInput.getValue() == null) {
                log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , file name shouldn't be null", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), processInput.getName());
                status = new TaskStatus(TaskState.FAILED);
                if (processInput.isIsRequired()) {
                    status.setReason("File name is null, but this input's isRequired bit is not set");
                } else {
                    status.setReason("File name is null");
                }
                return status;
            }
        } else {
            status.setState(TaskState.FAILED);
            status.setReason("Invalid task invocation, Support " + ProcessState.INPUT_DATA_STAGING.name() + " and " + "" + ProcessState.OUTPUT_DATA_STAGING.name() + " process phases. found " + processState.name());
            return status;
        }
        StorageResourceDescription storageResource = processContext.getStorageResource();
        // StoragePreference storagePreference = taskContext.getParentProcessContext().getStoragePreference();
        String hostName = null;
        if (storageResource != null) {
            hostName = storageResource.getHostName();
        } else {
            throw new GFacException("Storage Resource is null");
        }
        String inputPath = processContext.getStorageFileSystemRootLocation();
        inputPath = (inputPath.endsWith(File.separator) ? inputPath : inputPath + File.separator);
        // use rsync instead of scp if source and destination host and user name is same.
        URI sourceURI = new URI(subTaskModel.getSource());
        String fileName = sourceURI.getPath().substring(sourceURI.getPath().lastIndexOf(File.separator) + 1, sourceURI.getPath().length());
        Session remoteSession = Factory.getSSHSession(Factory.getComputerResourceSSHKeyAuthentication(processContext), processContext.getComputeResourceServerInfo());
        Session storageSession = Factory.getSSHSession(Factory.getStorageSSHKeyAuthentication(processContext), processContext.getStorageResourceServerInfo());
        URI destinationURI = null;
        if (subTaskModel.getDestination().startsWith("dummy")) {
            destinationURI = TaskUtils.getDestinationURI(taskContext, hostName, inputPath, fileName);
            subTaskModel.setDestination(destinationURI.toString());
        } else {
            destinationURI = new URI(subTaskModel.getDestination());
        }
        if (sourceURI.getHost().equalsIgnoreCase(destinationURI.getHost()) && sourceURI.getUserInfo().equalsIgnoreCase(destinationURI.getUserInfo())) {
            localDataCopy(taskContext, sourceURI, destinationURI);
            status.setState(TaskState.COMPLETED);
            status.setReason("Locally copied file using 'cp' command ");
            return status;
        }
        status = new TaskStatus(TaskState.COMPLETED);
        // Wildcard for file name. Has to find the correct name.
        if (fileName.contains("*")) {
            String destParentPath = (new File(destinationURI.getPath())).getParentFile().getPath();
            String sourceParentPath = (new File(sourceURI.getPath())).getParentFile().getPath();
            List<String> fileNames = taskContext.getParentProcessContext().getDataMovementRemoteCluster().getFileNameFromExtension(fileName, sourceParentPath, remoteSession);
            ExperimentCatalog experimentCatalog = processContext.getExperimentCatalog();
            String experimentId = processContext.getExperimentId();
            String processId = processContext.getProcessId();
            OutputDataObjectType processOutput = taskContext.getProcessOutput();
            for (int i = 0; i < fileNames.size(); i++) {
                String temp = fileNames.get(i);
                if (temp != null && temp != "") {
                    fileName = temp;
                }
                if (destParentPath.endsWith(File.separator)) {
                    destinationURI = new URI(destParentPath + fileName);
                } else {
                    destinationURI = new URI(destParentPath + File.separator + fileName);
                }
                // Wildcard support is only enabled for output data staging
                if (processState == ProcessState.OUTPUT_DATA_STAGING) {
                    processOutput.setName(fileName);
                    experimentCatalog.add(ExpCatChildDataType.EXPERIMENT_OUTPUT, Arrays.asList(processOutput), experimentId);
                    experimentCatalog.add(ExpCatChildDataType.PROCESS_OUTPUT, Arrays.asList(processOutput), processId);
                    taskContext.setProcessOutput(processOutput);
                    makeDir(taskContext, destinationURI);
                    // TODO - save updated subtask model with new destination
                    outputDataStaging(taskContext, remoteSession, sourceURI, storageSession, destinationURI);
                    status.setReason("Successfully staged output data");
                }
            }
            if (processState == ProcessState.OUTPUT_DATA_STAGING) {
                status.setReason("Successfully staged output data");
            } else {
                status.setReason("Wildcard support is only enabled for output data staging");
            }
        } else {
            if (processState == ProcessState.INPUT_DATA_STAGING) {
                inputDataStaging(taskContext, storageSession, sourceURI, remoteSession, destinationURI);
                status.setReason("Successfully staged input data");
            } else if (processState == ProcessState.OUTPUT_DATA_STAGING) {
                makeDir(taskContext, destinationURI);
                // TODO - save updated subtask model with new destination
                outputDataStaging(taskContext, remoteSession, sourceURI, storageSession, destinationURI);
                status.setReason("Successfully staged output data");
            }
        }
    } catch (TException e) {
        String msg = "Couldn't create subTask model thrift model";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
        return status;
    } catch (ApplicationSettingsException | FileNotFoundException e) {
        String msg = "Failed while reading credentials";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (URISyntaxException e) {
        String msg = "Source or destination uri is not correct source : " + subTaskModel.getSource() + ", " + "destination : " + subTaskModel.getDestination();
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (CredentialStoreException e) {
        String msg = "Storage authentication issue, could be invalid credential token";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (AiravataException e) {
        String msg = "Error while creating ssh session with client";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (JSchException | IOException e) {
        String msg = "Failed to do scp with client";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    } catch (RegistryException | GFacException e) {
        String msg = "Data staging failed";
        log.error(msg, e);
        status.setState(TaskState.FAILED);
        status.setReason(msg);
        ErrorModel errorModel = new ErrorModel();
        errorModel.setActualErrorMessage(e.getMessage());
        errorModel.setUserFriendlyMessage(msg);
        taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
    }
    return status;
}

Example 19 with GFacException

use of org.apache.airavata.gfac.core.GFacException in project airavata by apache.

the class StreamData method runOutputStaging.

public void runOutputStaging() throws URISyntaxException, IllegalAccessException, InstantiationException, CredentialStoreException, AiravataException, IOException, JSchException {
    try {
        URI sourceURI = new URI(subTaskModel.getSource());
        String fileName = sourceURI.getPath().substring(sourceURI.getPath().lastIndexOf(File.separator) + 1, sourceURI.getPath().length());
        URI destinationURI = null;
        if (subTaskModel.getDestination().startsWith("dummy")) {
            destinationURI = getDestinationURI(taskContext, fileName);
            subTaskModel.setDestination(destinationURI.toString());
        } else {
            destinationURI = new URI(subTaskModel.getDestination());
        }
        if (sourceURI.getHost().equalsIgnoreCase(destinationURI.getHost()) && sourceURI.getUserInfo().equalsIgnoreCase(destinationURI.getUserInfo())) {
            localDataCopy(taskContext, sourceURI, destinationURI);
        }
        Session srcSession = Factory.getSSHSession(Factory.getComputerResourceSSHKeyAuthentication(taskContext.getParentProcessContext()), taskContext.getParentProcessContext().getComputeResourceServerInfo());
        Session destSession = Factory.getSSHSession(Factory.getStorageSSHKeyAuthentication(taskContext.getParentProcessContext()), taskContext.getParentProcessContext().getStorageResourceServerInfo());
        String targetPath = destinationURI.getPath().substring(0, destinationURI.getPath().lastIndexOf('/'));
        SSHUtils.makeDirectory(targetPath, destSession);
        outputDataStaging(taskContext, srcSession, sourceURI, destSession, destinationURI);
    } catch (GFacException e) {
        log.error("expId: {}, processId:{}, taskId: {}:- Couldn't stage file {} , Error while output staging", taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), taskContext.getProcessOutput().getName());
        throw new AiravataException("Error while output staging", e);
    }
}

Example 20 with GFacException

use of org.apache.airavata.gfac.core.GFacException in project airavata by apache.

the class DataTransferrer method downloadRemoteFiles.

public List<OutputDataObjectType> downloadRemoteFiles() throws GFacException {
    if (log.isDebugEnabled()) {
        log.debug("Download location is:" + gatewayDownloadLocation);
    }
    List<OutputDataObjectType> applicationOutputs = processContext.getProcessModel().getProcessOutputs();
    if (applicationOutputs != null && !applicationOutputs.isEmpty()) {
        for (OutputDataObjectType output : applicationOutputs) {
            if ("".equals(output.getValue()) || output.getValue() == null) {
                continue;
            }
            if (output.getType().equals(DataType.STDOUT)) {
                output.setValue(stdoutLocation);
                resultantOutputsLst.add(output);
            } else if (output.getType().equals(DataType.STDERR)) {
                output.setValue(stderrLocation);
                resultantOutputsLst.add(output);
            } else if (output.getType().equals(DataType.URI)) {
                String value = null;
                if (!output.getLocation().isEmpty()) {
                    value = output.getLocation() + File.separator + output.getValue();
                } else {
                    value = output.getValue();
                }
                String outputPath = gatewayDownloadLocation + File.separator + output.getValue();
                File f = new File(gatewayDownloadLocation);
                if (!f.exists())
                    f.mkdirs();
                FileDownloader fileDownloader = new FileDownloader(value, outputPath, Mode.overwrite);
                try {
                    log.info("Downloading file {}", value);
                    fileDownloader.perform(storageClient);
                    output.setType(DataType.URI);
                    output.setValue(outputPath);
                    resultantOutputsLst.add(output);
                } catch (Exception e) {
                    log.error("Error downloading " + value + " from job working directory. ");
                // throw new GFacException(e.getLocalizedMessage(),e);
                }
            } else {
                log.info("Ignore output file {}, type {}", output.getValue(), output.getType().toString());
            }
        }
    }
    downloadStdOuts();
    return resultantOutputsLst;
}