Examples with JobDefinitionType org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType used on opensource projects

Example 1 with JobDefinitionType

use of org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType in project airavata by apache.

the class BESJobSubmissionTask method execute.

@Override
public TaskStatus execute(TaskContext taskContext) {
    TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
    StorageClient sc = null;
    ProcessContext processContext = taskContext.getParentProcessContext();
    // FIXME - use original output dir
    setInputOutputLocations(processContext);
    try {
        // con't reuse if UserDN has been changed.
        secProperties = getSecurityConfig(processContext);
    // try secProperties = secProperties.clone() if we can't use already initialized ClientConfigurations.
    } catch (GFacException e) {
        String msg = "Unicorn security context initialization error";
        log.error(msg, e);
        taskStatus.setState(TaskState.FAILED);
        taskStatus.setReason(msg);
        return taskStatus;
    }
    try {
        JobSubmissionProtocol protocol = processContext.getJobSubmissionProtocol();
        JobSubmissionInterface jobSubmissionInterface = GFacUtils.getPreferredJobSubmissionInterface(processContext);
        String factoryUrl = null;
        if (protocol.equals(JobSubmissionProtocol.UNICORE)) {
            UnicoreJobSubmission unicoreJobSubmission = GFacUtils.getUnicoreJobSubmission(jobSubmissionInterface.getJobSubmissionInterfaceId());
            factoryUrl = unicoreJobSubmission.getUnicoreEndPointURL();
        }
        EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
        eprt.addNewAddress().setStringValue(factoryUrl);
        String userDN = processContext.getProcessModel().getUserDn();
        CreateActivityDocument cad = CreateActivityDocument.Factory.newInstance();
        // create storage
        StorageCreator storageCreator = new StorageCreator(secProperties, factoryUrl, 5, null);
        sc = storageCreator.createStorage();
        JobDefinitionType jobDefinition = JSDLGenerator.buildJSDLInstance(processContext, sc.getUrl()).getJobDefinition();
        cad.addNewCreateActivity().addNewActivityDocument().setJobDefinition(jobDefinition);
        log.info("Submitted JSDL: " + jobDefinition.getJobDescription());
        // copy files to local
        copyInputFilesToLocal(taskContext);
        // upload files if any
        DataTransferrer dt = new DataTransferrer(processContext, sc);
        dt.uploadLocalFiles();
        JobModel jobDetails = new JobModel();
        jobDetails.setTaskId(taskContext.getTaskId());
        jobDetails.setProcessId(taskContext.getProcessId());
        FactoryClient factory = new FactoryClient(eprt, secProperties);
        log.info("Activity Submitting to {} ... \n", factoryUrl);
        CreateActivityResponseDocument response = factory.createActivity(cad);
        log.info("Activity Submitted to {} ... \n", factoryUrl);
        EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
        log.info("Activity : " + activityEpr.getAddress().getStringValue() + " Submitted.");
        // factory.waitWhileActivityIsDone(activityEpr, 1000);
        jobId = WSUtilities.extractResourceID(activityEpr);
        if (jobId == null) {
            jobId = new Long(Calendar.getInstance().getTimeInMillis()).toString();
        }
        log.info("JobID: " + jobId);
        jobDetails.setJobId(jobId);
        jobDetails.setJobDescription(activityEpr.toString());
        jobDetails.setJobStatuses(Arrays.asList(new JobStatus(JobState.SUBMITTED)));
        processContext.setJobModel(jobDetails);
        GFacUtils.saveJobModel(processContext, jobDetails);
        GFacUtils.saveJobStatus(processContext, jobDetails);
        log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(), factory.getActivityStatus(activityEpr).toString()));
        waitUntilDone(eprt, activityEpr, processContext, secProperties);
        ActivityStatusType activityStatus = null;
        activityStatus = getStatus(factory, activityEpr);
        log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(), activityStatus.getState().toString()));
        ActivityClient activityClient;
        activityClient = new ActivityClient(activityEpr, secProperties);
        // now use the activity working directory property
        dt.setStorageClient(activityClient.getUspaceClient());
        List<OutputDataObjectType> copyOutput = null;
        if ((activityStatus.getState() == ActivityStateEnumeration.FAILED)) {
            String error = activityStatus.getFault().getFaultcode().getLocalPart() + "\n" + activityStatus.getFault().getFaultstring() + "\n EXITCODE: " + activityStatus.getExitCode();
            log.error(error);
            JobState applicationJobStatus = JobState.FAILED;
            jobDetails.setJobStatuses(Arrays.asList(new JobStatus(applicationJobStatus)));
            sendNotification(processContext, jobDetails);
            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
            }
            // What if job is failed before execution and there are not stdouts generated yet?
            log.debug("Downloading any standard output and error files, if they were produced.");
            copyOutput = dt.downloadRemoteFiles();
        } else if (activityStatus.getState() == ActivityStateEnumeration.CANCELLED) {
            JobState applicationJobStatus = JobState.CANCELED;
            jobDetails.setJobStatuses(Arrays.asList(new JobStatus(applicationJobStatus)));
            GFacUtils.saveJobStatus(processContext, jobDetails);
            throw new GFacException(processContext.getExperimentId() + "Job Canceled");
        } else if (activityStatus.getState() == ActivityStateEnumeration.FINISHED) {
            try {
                Thread.sleep(5000);
            } catch (InterruptedException ignored) {
            }
            JobState applicationJobStatus = JobState.COMPLETE;
            jobDetails.setJobStatuses(Arrays.asList(new JobStatus(applicationJobStatus)));
            GFacUtils.saveJobStatus(processContext, jobDetails);
            log.info("Job Id: {}, exit code: {}, exit status: {}", jobDetails.getJobId(), activityStatus.getExitCode(), ActivityStateEnumeration.FINISHED.toString());
            // if (activityStatus.getExitCode() == 0) {
            // } else {
            // dt.downloadStdOuts();
            // }
            copyOutput = dt.downloadRemoteFiles();
        }
        if (copyOutput != null) {
            copyOutputFilesToStorage(taskContext, copyOutput);
            for (OutputDataObjectType outputDataObjectType : copyOutput) {
                GFacUtils.saveExperimentOutput(processContext, outputDataObjectType.getName(), outputDataObjectType.getValue());
            }
        }
        // dt.publishFinalOutputs();
        taskStatus.setState(TaskState.COMPLETED);
    } catch (AppCatalogException e) {
        log.error("Error while retrieving UNICORE job submission..", e);
        taskStatus.setState(TaskState.FAILED);
    } catch (Exception e) {
        log.error("BES task failed... ", e);
        taskStatus.setState(TaskState.FAILED);
    }
    return taskStatus;
}

Example 2 with JobDefinitionType

use of org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType in project airavata by apache.

the class ApplicationProcessor method createGenericApplication.

public static void createGenericApplication(JobDefinitionType value, String appName) {
    ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
    appType.setApplicationName(appName);
}

Example 3 with JobDefinitionType

use of org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType in project airavata by apache.

the class ApplicationProcessor method generateJobSpecificAppElements.

public static void generateJobSpecificAppElements(JobDefinitionType value, ProcessContext context) {
    String userName = getUserNameFromContext(context);
    // if (userName.equalsIgnoreCase("admin")){
    // userName = "CN=zdv575, O=Ultrascan Gateway, C=DE";
    // }
    ApplicationDeploymentDescription appDep = context.getApplicationDeploymentDescription();
    String appname = context.getApplicationInterfaceDescription().getApplicationName();
    ApplicationParallelismType parallelism = appDep.getParallelism();
    ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
    appType.setApplicationName(appname);
    // if (appDep.getSetEnvironment().size() > 0) {
    // createApplicationEnvironment(value, appDep.getSetEnvironment(), parallelism);
    // }
    // 
    String stdout = context.getStdoutLocation();
    String stderr = context.getStderrLocation();
    if (stdout != null) {
        stdout = stdout.substring(stdout.lastIndexOf('/') + 1);
    }
    if (stderr != null) {
        stderr = stderr.substring(stderr.lastIndexOf('/') + 1);
    }
    stdout = (stdout == null || stdout.equals("")) ? "stdout" : stdout;
    stderr = (stdout == null || stderr.equals("")) ? "stderr" : stderr;
    if (appDep.getExecutablePath() != null) {
        FileNameType fNameType = FileNameType.Factory.newInstance();
        fNameType.setStringValue(appDep.getExecutablePath());
        if (isParallelJob(context)) {
            JSDLUtils.getOrCreateSPMDApplication(value).setExecutable(fNameType);
            if (parallelism.equals(ApplicationParallelismType.OPENMP_MPI)) {
                JSDLUtils.getSPMDApplication(value).setSPMDVariation(SPMDVariations.OpenMPI.value());
            } else if (parallelism.equals(ApplicationParallelismType.MPI)) {
                JSDLUtils.getSPMDApplication(value).setSPMDVariation(SPMDVariations.MPI.value());
            }
            // setting number of processes
            try {
                String np = getInputAsString(context, BESConstants.NUMBER_OF_PROCESSES);
                if ((np != null) && (Integer.parseInt(np) > 0)) {
                    NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
                    num.setStringValue(np);
                    JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num);
                }
            } catch (RuntimeException np) {
            // do nothing
            }
            try {
                // setting processes per host
                String pphost = getInputAsString(context, BESConstants.PROCESSES_PER_HOST);
                if ((pphost != null) && (Integer.parseInt(pphost) > 0)) {
                    ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance();
                    pph.setStringValue(String.valueOf(pphost));
                    JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph);
                }
            } catch (RuntimeException np) {
            // do nothing
            }
            int totalThreadCount = context.getProcessModel().getProcessResourceSchedule().getNumberOfThreads();
            // we take it as threads per processes
            if (totalThreadCount > 0) {
                ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance();
                tpp.setStringValue(String.valueOf(totalThreadCount));
                JSDLUtils.getSPMDApplication(value).setThreadsPerProcess(tpp);
            }
            if (userName != null) {
                UserNameType userNameType = UserNameType.Factory.newInstance();
                userNameType.setStringValue(userName);
                JSDLUtils.getSPMDApplication(value).setUserName(userNameType);
            }
            if (stdout != null) {
                FileNameType fName = FileNameType.Factory.newInstance();
                fName.setStringValue(stdout);
                JSDLUtils.getOrCreateSPMDApplication(value).setOutput(fName);
            }
            if (stderr != null) {
                FileNameType fName = FileNameType.Factory.newInstance();
                fName.setStringValue(stderr);
                JSDLUtils.getOrCreateSPMDApplication(value).setError(fName);
            }
        } else {
            JSDLUtils.getOrCreatePOSIXApplication(value).setExecutable(fNameType);
            if (userName != null) {
                UserNameType userNameType = UserNameType.Factory.newInstance();
                userNameType.setStringValue(userName);
                JSDLUtils.getOrCreatePOSIXApplication(value).setUserName(userNameType);
            }
            if (stdout != null) {
                FileNameType fName = FileNameType.Factory.newInstance();
                fName.setStringValue(stdout);
                JSDLUtils.getOrCreatePOSIXApplication(value).setOutput(fName);
            }
            if (stderr != null) {
                FileNameType fName = FileNameType.Factory.newInstance();
                fName.setStringValue(stderr);
                JSDLUtils.getOrCreatePOSIXApplication(value).setError(fName);
            }
        }
    }
}

Example 4 with JobDefinitionType

use of org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType in project airavata by apache.

the class JSDLGenerator method buildJSDLInstance.

public static synchronized JobDefinitionDocument buildJSDLInstance(ProcessContext context, String smsUrl, Object jobDirectoryMode) throws Exception {
    JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory.newInstance();
    JobDefinitionType value = jobDefDoc.addNewJobDefinition();
    // build Identification
    createJobIdentification(value, context);
    ResourceProcessor.generateResourceElements(value, context);
    ApplicationProcessor.generateJobSpecificAppElements(value, context);
    UASDataStagingProcessor.generateDataStagingElements(value, context, smsUrl);
    return jobDefDoc;
}

Example 5 with JobDefinitionType

use of org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType in project airavata by apache.

the class JSDLGenerator method buildJSDLInstance.

public static synchronized JobDefinitionDocument buildJSDLInstance(ProcessContext context) throws Exception {
    JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory.newInstance();
    JobDefinitionType value = jobDefDoc.addNewJobDefinition();
    // build Identification
    createJobIdentification(value, context);
    ResourceProcessor.generateResourceElements(value, context);
    ApplicationProcessor.generateJobSpecificAppElements(value, context);
    return jobDefDoc;
}