Examples with ApplicationDeploymentDescription org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription used on opensource projects

Example 6 with ApplicationDeploymentDescription

use of org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription in project airavata by apache.

the class DataTransferrer method downloadStdOuts.

public void downloadStdOuts() throws GFacProviderException {
    String downloadLocation = getDownloadLocation();
    File file = new File(downloadLocation);
    if (!file.exists()) {
        file.mkdirs();
    }
    String stdout = jobContext.getStandardOutput();
    String stderr = jobContext.getStandardError();
    if (stdout != null) {
        stdout = stdout.substring(stdout.lastIndexOf('/') + 1);
    }
    if (stderr != null) {
        stderr = stderr.substring(stderr.lastIndexOf('/') + 1);
    }
    String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout" : stdout;
    String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr" : stderr;
    ApplicationDeploymentDescription application = jobContext.getApplicationContext().getApplicationDeploymentDescription();
    String stdoutLocation = downloadLocation + File.separator + stdoutFileName;
    FileDownloader f1 = new FileDownloader(stdoutFileName, stdoutLocation, Mode.overwrite);
    try {
        f1.perform(storageClient);
        log.info("Downloading stdout and stderr..");
        String stdoutput = readFile(stdoutLocation);
        jobContext.addOutputFile(stdoutLocation);
        jobContext.setStandardOutput(stdoutLocation);
        log.info("Stdout downloaded to -> " + stdoutLocation);
        if (UASDataStagingProcessor.isUnicoreEndpoint(jobContext)) {
            String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
            String scriptCodeLocation = downloadLocation + File.separator + scriptExitCodeFName;
            f1.setFrom(scriptExitCodeFName);
            f1.setTo(scriptCodeLocation);
            f1.perform(storageClient);
            log.info("UNICORE_SCRIPT_EXIT_CODE downloaded to " + scriptCodeLocation);
        }
        String stderrLocation = downloadLocation + File.separator + stderrFileName;
        f1.setFrom(stderrFileName);
        f1.setTo(stderrLocation);
        f1.perform(storageClient);
        String stderror = readFile(stderrLocation);
        jobContext.addOutputFile(stderrLocation);
        jobContext.setStandardError(stderrLocation);
        log.info("Stderr downloaded to -> " + stderrLocation);
    } catch (Exception e) {
        throw new GFacProviderException(e.getLocalizedMessage(), e);
    }
}

Example 7 with ApplicationDeploymentDescription

use of org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription in project airavata by apache.

the class GFacUtils method createGroovyMap.

public static GroovyMap createGroovyMap(ProcessContext processContext, TaskContext taskContext) throws GFacException, AppCatalogException, ApplicationSettingsException {
    GroovyMap groovyMap = new GroovyMap();
    ProcessModel processModel = processContext.getProcessModel();
    ResourceJobManager resourceJobManager = getResourceJobManager(processContext);
    // set email options and addresses
    setMailAddresses(processContext, groovyMap);
    groovyMap.add(Script.INPUT_DIR, processContext.getInputDir());
    groovyMap.add(Script.OUTPUT_DIR, processContext.getOutputDir());
    groovyMap.add(Script.EXECUTABLE_PATH, processContext.getApplicationDeploymentDescription().getExecutablePath());
    groovyMap.add(Script.STANDARD_OUT_FILE, processContext.getStdoutLocation());
    groovyMap.add(Script.STANDARD_ERROR_FILE, processContext.getStderrLocation());
    groovyMap.add(Script.SCRATCH_LOCATION, processContext.getScratchLocation());
    groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
    groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName());
    groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName());
    groovyMap.add(Script.QUEUE_SPECIFIC_MACROS, processContext.getQueueSpecificMacros());
    groovyMap.add(Script.ACCOUNT_STRING, processContext.getAllocationProjectNumber());
    groovyMap.add(Script.RESERVATION, processContext.getReservation());
    // To make job name alpha numeric
    groovyMap.add(Script.JOB_NAME, "A" + String.valueOf(generateJobName()));
    groovyMap.add(Script.WORKING_DIR, processContext.getWorkingDir());
    List<String> inputValues = getProcessInputValues(processModel.getProcessInputs());
    inputValues.addAll(getProcessOutputValues(processModel.getProcessOutputs()));
    groovyMap.add(Script.INPUTS, inputValues);
    groovyMap.add(Script.USER_NAME, processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName());
    groovyMap.add(Script.SHELL_NAME, "/bin/bash");
    // get walltime
    if (taskContext != null) {
        try {
            JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel());
            if (jobSubmissionTaskModel.getWallTime() > 0) {
                groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
                if (resourceJobManager != null) {
                    if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                        groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(jobSubmissionTaskModel.getWallTime()));
                    }
                }
            }
        } catch (TException e) {
            log.error("Error while getting job submission sub task model", e);
        }
    }
    // NOTE: Give precedence to data comes with experiment
    // qos per queue
    String qoS = getQoS(processContext.getQualityOfService(), processContext.getQueueName());
    if (qoS != null) {
        groovyMap.add(Script.QUALITY_OF_SERVICE, qoS);
    }
    ComputationalResourceSchedulingModel scheduling = processModel.getProcessResourceSchedule();
    if (scheduling != null) {
        int totalNodeCount = scheduling.getNodeCount();
        int totalCPUCount = scheduling.getTotalCPUCount();
        if (isValid(scheduling.getQueueName())) {
            groovyMap.add(Script.QUEUE_NAME, scheduling.getQueueName());
        }
        if (totalNodeCount > 0) {
            groovyMap.add(Script.NODES, totalNodeCount);
        }
        if (totalCPUCount > 0) {
            int ppn = totalCPUCount / totalNodeCount;
            groovyMap.add(Script.PROCESS_PER_NODE, ppn);
            groovyMap.add(Script.CPU_COUNT, totalCPUCount);
        }
        // if so we ignore scheduling configuration.
        if (scheduling.getWallTimeLimit() > 0 && groovyMap.get(Script.MAX_WALL_TIME) == null) {
            groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(scheduling.getWallTimeLimit()));
            if (resourceJobManager != null) {
                if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                    groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(scheduling.getWallTimeLimit()));
                }
            }
        }
        if (scheduling.getTotalPhysicalMemory() > 0) {
            groovyMap.add(Script.USED_MEM, scheduling.getTotalPhysicalMemory());
        }
        if (isValid(scheduling.getOverrideLoginUserName())) {
            groovyMap.add(Script.USER_NAME, scheduling.getOverrideLoginUserName());
        }
        if (isValid(scheduling.getOverrideAllocationProjectNumber())) {
            groovyMap.add(Script.ACCOUNT_STRING, scheduling.getOverrideAllocationProjectNumber());
        }
        if (isValid(scheduling.getStaticWorkingDir())) {
            groovyMap.add(Script.WORKING_DIR, scheduling.getStaticWorkingDir());
        }
    } else {
        log.error("Task scheduling cannot be null at this point..");
    }
    ApplicationDeploymentDescription appDepDescription = processContext.getApplicationDeploymentDescription();
    List<CommandObject> moduleCmds = appDepDescription.getModuleLoadCmds();
    if (moduleCmds != null) {
        List<String> modulesCmdCollect = moduleCmds.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> map.getCommand()).collect(Collectors.toList());
        groovyMap.add(Script.MODULE_COMMANDS, modulesCmdCollect);
    }
    List<CommandObject> preJobCommands = appDepDescription.getPreJobCommands();
    if (preJobCommands != null) {
        List<String> preJobCmdCollect = preJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect);
    }
    List<CommandObject> postJobCommands = appDepDescription.getPostJobCommands();
    if (postJobCommands != null) {
        List<String> postJobCmdCollect = postJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect);
    }
    ApplicationParallelismType parallelism = appDepDescription.getParallelism();
    if (parallelism != null) {
        if (parallelism != ApplicationParallelismType.SERIAL) {
            Map<ApplicationParallelismType, String> parallelismPrefix = processContext.getResourceJobManager().getParallelismPrefix();
            if (parallelismPrefix != null) {
                String parallelismCommand = parallelismPrefix.get(parallelism);
                if (parallelismCommand != null) {
                    groovyMap.add(Script.JOB_SUBMITTER_COMMAND, parallelismCommand);
                } else {
                    throw new GFacException("Parallelism prefix is not defined for given parallelism type " + parallelism + ".. Please define the parallelism prefix at App Catalog");
                }
            }
        }
    }
    return groovyMap;
}

Example 8 with ApplicationDeploymentDescription

use of org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription in project airavata by apache.

the class RegistryServerHandler method getAppModuleDeployedResources.

/**
 * Fetch a list of Deployed Compute Hosts.
 *
 * @param appModuleId The identifier for the requested application module
 * @return list<string>
 * Returns a list of Deployed Resources.
 */
@Override
public List<String> getAppModuleDeployedResources(String appModuleId) throws RegistryServiceException, TException {
    try {
        List<String> appDeployments = new ArrayList<String>();
        appCatalog = RegistryFactory.getAppCatalog();
        Map<String, String> filters = new HashMap<String, String>();
        filters.put(AppCatAbstractResource.ApplicationDeploymentConstants.APP_MODULE_ID, appModuleId);
        List<ApplicationDeploymentDescription> applicationDeployments = appCatalog.getApplicationDeployment().getApplicationDeployements(filters);
        for (ApplicationDeploymentDescription description : applicationDeployments) {
            appDeployments.add(description.getAppDeploymentId());
        }
        logger.debug("Airavata retrieved application deployments for module id : " + appModuleId);
        return appDeployments;
    } catch (AppCatalogException e) {
        logger.error(appModuleId, "Error while retrieving application deployments...", e);
        RegistryServiceException exception = new RegistryServiceException();
        exception.setMessage("Error while retrieving application deployment. More info : " + e.getMessage());
        throw exception;
    }
}

Example 9 with ApplicationDeploymentDescription

use of org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription in project airavata by apache.

the class RegistryServerHandler method getAllApplicationDeployments.

/**
 * Fetch all Application Deployment Descriptions.
 *
 * @param gatewayId ID of the gateway which need to list all available application deployment documentation.
 * @param gatewayId
 * @return list<applicationDeployment.
 * Returns the list of all application Deployment Objects.
 */
@Override
public List<ApplicationDeploymentDescription> getAllApplicationDeployments(String gatewayId) throws RegistryServiceException, TException {
    if (!isGatewayExistInternal(gatewayId)) {
        logger.error("Gateway does not exist.Please provide a valid gateway id...");
        throw new RegistryServiceException("Gateway does not exist.Please provide a valid gateway id...");
    }
    try {
        appCatalog = RegistryFactory.getAppCatalog();
        List<ApplicationDeploymentDescription> deployements = appCatalog.getApplicationDeployment().getAllApplicationDeployements(gatewayId);
        logger.debug("Airavata retrieved application deployments for gateway id : " + gatewayId);
        return deployements;
    } catch (AppCatalogException e) {
        logger.error("Error while retrieving application deployments...", e);
        RegistryServiceException exception = new RegistryServiceException();
        exception.setMessage("Error while retrieving application deployments. More info : " + e.getMessage());
        throw exception;
    }
}

Example 10 with ApplicationDeploymentDescription

use of org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription in project airavata by apache.

the class RegistryServerHandler method getAvailableAppInterfaceComputeResources.

/**
 * Fetch a list of all deployed Compute Hosts for a given application interfaces.
 *
 * @param appInterfaceId The identifier for the requested application interface.
 * @return map<computeResourceId, computeResourceName>
 * A map of registered compute resource id's and their corresponding hostnames.
 * Deployments of each modules listed within the interfaces will be listed.
 */
@Override
public Map<String, String> getAvailableAppInterfaceComputeResources(String appInterfaceId) throws RegistryServiceException, TException {
    try {
        appCatalog = RegistryFactory.getAppCatalog();
        ApplicationDeployment applicationDeployment = appCatalog.getApplicationDeployment();
        Map<String, String> allComputeResources = appCatalog.getComputeResource().getAvailableComputeResourceIdList();
        Map<String, String> availableComputeResources = new HashMap<String, String>();
        ApplicationInterfaceDescription applicationInterface = appCatalog.getApplicationInterface().getApplicationInterface(appInterfaceId);
        HashMap<String, String> filters = new HashMap<String, String>();
        List<String> applicationModules = applicationInterface.getApplicationModules();
        if (applicationModules != null && !applicationModules.isEmpty()) {
            for (String moduleId : applicationModules) {
                filters.put(AppCatAbstractResource.ApplicationDeploymentConstants.APP_MODULE_ID, moduleId);
                List<ApplicationDeploymentDescription> applicationDeployments = applicationDeployment.getApplicationDeployements(filters);
                for (ApplicationDeploymentDescription deploymentDescription : applicationDeployments) {
                    if (allComputeResources.get(deploymentDescription.getComputeHostId()) != null) {
                        availableComputeResources.put(deploymentDescription.getComputeHostId(), allComputeResources.get(deploymentDescription.getComputeHostId()));
                    }
                }
            }
        }
        logger.debug("Airavata retrieved available compute resources for application interface id : " + appInterfaceId);
        return availableComputeResources;
    } catch (AppCatalogException e) {
        logger.error(appInterfaceId, "Error while saving compute resource...", e);
        RegistryServiceException exception = new RegistryServiceException();
        exception.setMessage("Error while saving compute resource. More info : " + e.getMessage());
        throw exception;
    }
}