Examples with TaskContext org.apache.airavata.gfac.core.context.TaskContext used on opensource projects

Example 1 with TaskContext

use of org.apache.airavata.gfac.core.context.TaskContext in project airavata by apache.

the class GFacEngineImpl method getJobSubmissionTaskContext.

private TaskContext getJobSubmissionTaskContext(ProcessContext processContext) throws GFacException {
    TaskContext taskCtx = new TaskContext();
    taskCtx.setParentProcessContext(processContext);
    TaskModel taskModel = new TaskModel();
    taskModel.setParentProcessId(processContext.getProcessId());
    taskModel.setCreationTime(new Date().getTime());
    taskModel.setLastUpdateTime(taskModel.getCreationTime());
    TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
    taskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
    taskModel.setTaskStatuses(Arrays.asList(taskStatus));
    taskModel.setTaskType(TaskTypes.JOB_SUBMISSION);
    taskCtx.setTaskModel(taskModel);
    return taskCtx;
}

Example 2 with TaskContext

use of org.apache.airavata.gfac.core.context.TaskContext in project airavata by apache.

the class GFacEngineImpl method executeTaskListFrom.

private void executeTaskListFrom(ProcessContext processContext, String startingTaskId) throws GFacException {
    // checkpoint
    if (processContext.isInterrupted() && processContext.getProcessState() != ProcessState.MONITORING) {
        GFacUtils.handleProcessInterrupt(processContext);
        return;
    }
    List<TaskModel> taskList = processContext.getTaskList();
    Map<String, TaskModel> taskMap = processContext.getTaskMap();
    boolean fastForward = true;
    for (String taskId : processContext.getTaskExecutionOrder()) {
        if (fastForward) {
            if (taskId.equalsIgnoreCase(startingTaskId)) {
                fastForward = false;
            } else {
                continue;
            }
        }
        TaskModel taskModel = taskMap.get(taskId);
        processContext.setCurrentExecutingTaskModel(taskModel);
        TaskTypes taskType = taskModel.getTaskType();
        TaskContext taskContext = getTaskContext(processContext);
        taskContext.setTaskModel(taskModel);
        ProcessStatus status = null;
        switch(taskType) {
            case ENV_SETUP:
                status = new ProcessStatus(ProcessState.CONFIGURING_WORKSPACE);
                status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
                processContext.setProcessStatus(status);
                GFacUtils.saveAndPublishProcessStatus(processContext);
                // checkpoint
                if (processContext.isInterrupted()) {
                    GFacUtils.handleProcessInterrupt(processContext);
                    return;
                }
                configureWorkspace(taskContext, processContext.isRecovery());
                // checkpoint
                if (processContext.isInterrupted()) {
                    GFacUtils.handleProcessInterrupt(processContext);
                    return;
                }
                break;
            case DATA_STAGING:
                try {
                    // checkpoint
                    if (processContext.isInterrupted()) {
                        GFacUtils.handleProcessInterrupt(processContext);
                        return;
                    }
                    DataStagingTaskModel subTaskModel = (DataStagingTaskModel) taskContext.getSubTaskModel();
                    DataStageType type = subTaskModel.getType();
                    switch(type) {
                        case INPUT:
                            status = new ProcessStatus(ProcessState.INPUT_DATA_STAGING);
                            status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
                            processContext.setProcessStatus(status);
                            GFacUtils.saveAndPublishProcessStatus(processContext);
                            taskContext.setProcessInput(subTaskModel.getProcessInput());
                            inputDataStaging(taskContext, processContext.isRecovery());
                            break;
                        case OUPUT:
                            status = new ProcessStatus(ProcessState.OUTPUT_DATA_STAGING);
                            status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
                            processContext.setProcessStatus(status);
                            GFacUtils.saveAndPublishProcessStatus(processContext);
                            taskContext.setProcessOutput(subTaskModel.getProcessOutput());
                            outputDataStaging(taskContext, processContext.isRecovery(), false);
                            break;
                        case ARCHIVE_OUTPUT:
                            status = new ProcessStatus(ProcessState.OUTPUT_DATA_STAGING);
                            status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
                            processContext.setProcessStatus(status);
                            GFacUtils.saveAndPublishProcessStatus(processContext);
                            outputDataStaging(taskContext, processContext.isRecovery(), true);
                            break;
                    }
                    // checkpoint
                    if (processContext.isInterrupted()) {
                        GFacUtils.handleProcessInterrupt(processContext);
                        return;
                    }
                } catch (TException e) {
                    throw new GFacException(e);
                }
                break;
            case JOB_SUBMISSION:
                // checkpoint
                if (processContext.isInterrupted()) {
                    GFacUtils.handleProcessInterrupt(processContext);
                    return;
                }
                status = new ProcessStatus(ProcessState.EXECUTING);
                status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
                processContext.setProcessStatus(status);
                GFacUtils.saveAndPublishProcessStatus(processContext);
                executeJobSubmission(taskContext, processContext.isRecovery());
                // Don't put any checkpoint in between JobSubmission and Monitoring tasks
                JobStatus jobStatus = processContext.getJobModel().getJobStatuses().get(0);
                if (jobStatus != null && (jobStatus.getJobState() == JobState.SUBMITTED || jobStatus.getJobState() == JobState.QUEUED || jobStatus.getJobState() == JobState.ACTIVE)) {
                    List<OutputDataObjectType> processOutputs = processContext.getProcessModel().getProcessOutputs();
                    if (processOutputs != null && !processOutputs.isEmpty()) {
                        for (OutputDataObjectType output : processOutputs) {
                            try {
                                if (output.isOutputStreaming()) {
                                    TaskModel streamingTaskModel = new TaskModel();
                                    streamingTaskModel.setTaskType(TaskTypes.OUTPUT_FETCHING);
                                    streamingTaskModel.setTaskStatuses(Arrays.asList(new TaskStatus(TaskState.CREATED)));
                                    streamingTaskModel.setCreationTime(AiravataUtils.getCurrentTimestamp().getTime());
                                    streamingTaskModel.setParentProcessId(processContext.getProcessId());
                                    TaskContext streamingTaskContext = getTaskContext(processContext);
                                    DataStagingTaskModel submodel = new DataStagingTaskModel();
                                    submodel.setType(DataStageType.OUPUT);
                                    submodel.setProcessOutput(output);
                                    URI source = new URI(processContext.getDataMovementProtocol().name(), processContext.getComputeResourceLoginUserName(), processContext.getComputeResourceDescription().getHostName(), 22, processContext.getWorkingDir() + output.getValue(), null, null);
                                    submodel.setSource(source.getPath());
                                    submodel.setDestination("dummy://temp/file/location");
                                    streamingTaskModel.setSubTaskModel(ThriftUtils.serializeThriftObject(submodel));
                                    String streamTaskId = (String) processContext.getExperimentCatalog().add(ExpCatChildDataType.TASK, streamingTaskModel, processContext.getProcessId());
                                    streamingTaskModel.setTaskId(streamTaskId);
                                    streamingTaskContext.setTaskModel(streamingTaskModel);
                                    executeDataStreaming(streamingTaskContext, processContext.isRecovery());
                                }
                            } catch (URISyntaxException | TException | RegistryException e) {
                                log.error("Error while streaming output " + output.getValue());
                            }
                        }
                    }
                }
                break;
            case MONITORING:
                status = new ProcessStatus(ProcessState.MONITORING);
                status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
                processContext.setProcessStatus(status);
                GFacUtils.saveAndPublishProcessStatus(processContext);
                executeJobMonitoring(taskContext, processContext.isRecovery());
                break;
            case ENV_CLEANUP:
                // TODO implement environment clean up task logic
                break;
            default:
                throw new GFacException("Unsupported Task type");
        }
        if (processContext.isPauseTaskExecution()) {
            // If any task put processContext to wait, the same task must continue processContext execution.
            return;
        }
    }
    processContext.setComplete(true);
}

Example 3 with TaskContext

use of org.apache.airavata.gfac.core.context.TaskContext in project airavata by apache.

the class GFacUtils method createGroovyMap.

public static GroovyMap createGroovyMap(ProcessContext processContext, TaskContext taskContext) throws GFacException, AppCatalogException, ApplicationSettingsException {
    GroovyMap groovyMap = new GroovyMap();
    ProcessModel processModel = processContext.getProcessModel();
    ResourceJobManager resourceJobManager = getResourceJobManager(processContext);
    // set email options and addresses
    setMailAddresses(processContext, groovyMap);
    groovyMap.add(Script.INPUT_DIR, processContext.getInputDir());
    groovyMap.add(Script.OUTPUT_DIR, processContext.getOutputDir());
    groovyMap.add(Script.EXECUTABLE_PATH, processContext.getApplicationDeploymentDescription().getExecutablePath());
    groovyMap.add(Script.STANDARD_OUT_FILE, processContext.getStdoutLocation());
    groovyMap.add(Script.STANDARD_ERROR_FILE, processContext.getStderrLocation());
    groovyMap.add(Script.SCRATCH_LOCATION, processContext.getScratchLocation());
    groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
    groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName());
    groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName());
    groovyMap.add(Script.QUEUE_SPECIFIC_MACROS, processContext.getQueueSpecificMacros());
    groovyMap.add(Script.ACCOUNT_STRING, processContext.getAllocationProjectNumber());
    groovyMap.add(Script.RESERVATION, processContext.getReservation());
    // To make job name alpha numeric
    groovyMap.add(Script.JOB_NAME, "A" + String.valueOf(generateJobName()));
    groovyMap.add(Script.WORKING_DIR, processContext.getWorkingDir());
    List<String> inputValues = getProcessInputValues(processModel.getProcessInputs());
    inputValues.addAll(getProcessOutputValues(processModel.getProcessOutputs()));
    groovyMap.add(Script.INPUTS, inputValues);
    groovyMap.add(Script.USER_NAME, processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName());
    groovyMap.add(Script.SHELL_NAME, "/bin/bash");
    // get walltime
    if (taskContext != null) {
        try {
            JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel());
            if (jobSubmissionTaskModel.getWallTime() > 0) {
                groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
                if (resourceJobManager != null) {
                    if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                        groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(jobSubmissionTaskModel.getWallTime()));
                    }
                }
            }
        } catch (TException e) {
            log.error("Error while getting job submission sub task model", e);
        }
    }
    // NOTE: Give precedence to data comes with experiment
    // qos per queue
    String qoS = getQoS(processContext.getQualityOfService(), processContext.getQueueName());
    if (qoS != null) {
        groovyMap.add(Script.QUALITY_OF_SERVICE, qoS);
    }
    ComputationalResourceSchedulingModel scheduling = processModel.getProcessResourceSchedule();
    if (scheduling != null) {
        int totalNodeCount = scheduling.getNodeCount();
        int totalCPUCount = scheduling.getTotalCPUCount();
        if (isValid(scheduling.getQueueName())) {
            groovyMap.add(Script.QUEUE_NAME, scheduling.getQueueName());
        }
        if (totalNodeCount > 0) {
            groovyMap.add(Script.NODES, totalNodeCount);
        }
        if (totalCPUCount > 0) {
            int ppn = totalCPUCount / totalNodeCount;
            groovyMap.add(Script.PROCESS_PER_NODE, ppn);
            groovyMap.add(Script.CPU_COUNT, totalCPUCount);
        }
        // if so we ignore scheduling configuration.
        if (scheduling.getWallTimeLimit() > 0 && groovyMap.get(Script.MAX_WALL_TIME) == null) {
            groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculator(scheduling.getWallTimeLimit()));
            if (resourceJobManager != null) {
                if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) {
                    groovyMap.add(Script.MAX_WALL_TIME, GFacUtils.maxWallTimeCalculatorForLSF(scheduling.getWallTimeLimit()));
                }
            }
        }
        if (scheduling.getTotalPhysicalMemory() > 0) {
            groovyMap.add(Script.USED_MEM, scheduling.getTotalPhysicalMemory());
        }
        if (isValid(scheduling.getOverrideLoginUserName())) {
            groovyMap.add(Script.USER_NAME, scheduling.getOverrideLoginUserName());
        }
        if (isValid(scheduling.getOverrideAllocationProjectNumber())) {
            groovyMap.add(Script.ACCOUNT_STRING, scheduling.getOverrideAllocationProjectNumber());
        }
        if (isValid(scheduling.getStaticWorkingDir())) {
            groovyMap.add(Script.WORKING_DIR, scheduling.getStaticWorkingDir());
        }
    } else {
        log.error("Task scheduling cannot be null at this point..");
    }
    ApplicationDeploymentDescription appDepDescription = processContext.getApplicationDeploymentDescription();
    List<CommandObject> moduleCmds = appDepDescription.getModuleLoadCmds();
    if (moduleCmds != null) {
        List<String> modulesCmdCollect = moduleCmds.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> map.getCommand()).collect(Collectors.toList());
        groovyMap.add(Script.MODULE_COMMANDS, modulesCmdCollect);
    }
    List<CommandObject> preJobCommands = appDepDescription.getPreJobCommands();
    if (preJobCommands != null) {
        List<String> preJobCmdCollect = preJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect);
    }
    List<CommandObject> postJobCommands = appDepDescription.getPostJobCommands();
    if (postJobCommands != null) {
        List<String> postJobCmdCollect = postJobCommands.stream().sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()).map(map -> parseCommands(map.getCommand(), groovyMap)).collect(Collectors.toList());
        groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect);
    }
    ApplicationParallelismType parallelism = appDepDescription.getParallelism();
    if (parallelism != null) {
        if (parallelism != ApplicationParallelismType.SERIAL) {
            Map<ApplicationParallelismType, String> parallelismPrefix = processContext.getResourceJobManager().getParallelismPrefix();
            if (parallelismPrefix != null) {
                String parallelismCommand = parallelismPrefix.get(parallelism);
                if (parallelismCommand != null) {
                    groovyMap.add(Script.JOB_SUBMITTER_COMMAND, parallelismCommand);
                } else {
                    throw new GFacException("Parallelism prefix is not defined for given parallelism type " + parallelism + ".. Please define the parallelism prefix at App Catalog");
                }
            }
        }
    }
    return groovyMap;
}

Example 4 with TaskContext

use of org.apache.airavata.gfac.core.context.TaskContext in project airavata by apache.

the class EmailBasedMonitor method processMessages.

private void processMessages(Message[] searchMessages) throws MessagingException {
    List<Message> processedMessages = new ArrayList<>();
    List<Message> unreadMessages = new ArrayList<>();
    for (Message message : searchMessages) {
        try {
            JobStatusResult jobStatusResult = parse(message);
            TaskContext taskContext = null;
            if (jobStatusResult.getJobId() != null) {
                taskContext = jobMonitorMap.get(jobStatusResult.getJobId());
            } else {
                log.info("Returned null for job id, message subject--> {}", message.getSubject());
            }
            if (taskContext == null) {
                if (jobStatusResult.getJobName() != null) {
                    taskContext = jobMonitorMap.get(jobStatusResult.getJobName());
                } else {
                    log.info("Returned null for job name, message subject --> {}", message.getSubject());
                }
            }
            if (taskContext != null) {
                process(jobStatusResult, taskContext);
                processedMessages.add(message);
            } else if (!jobStatusResult.isAuthoritative() && (new Date()).getTime() - message.getSentDate().getTime() > 1000 * 60 * 5) {
                // marking old custom Airavata emails as read
                processedMessages.add(message);
                log.info("Marking old Airavata custom emails as read, message subject --> {}", message.getSubject());
            } else {
                // we can get JobExecutionContext null in multiple Gfac instances environment,
                // where this job is not submitted by this Gfac instance hence we ignore this message.
                unreadMessages.add(message);
            // log.info("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId());
            }
        } catch (AiravataException e) {
            log.error("[EJM]: Error parsing email message =====================================>", e);
            try {
                writeEnvelopeOnError(message);
            } catch (MessagingException e1) {
                log.error("[EJM]: Error printing envelop of the email");
            }
            unreadMessages.add(message);
        } catch (MessagingException e) {
            log.error("[EJM]: Error while retrieving sender address from message : " + message.toString());
            unreadMessages.add(message);
        }
    }
    if (!processedMessages.isEmpty()) {
        Message[] seenMessages = new Message[processedMessages.size()];
        processedMessages.toArray(seenMessages);
        try {
            emailFolder.setFlags(seenMessages, new Flags(Flags.Flag.SEEN), true);
        } catch (MessagingException e) {
            if (!store.isConnected()) {
                store.connect();
                emailFolder.setFlags(seenMessages, new Flags(Flags.Flag.SEEN), true);
            }
        }
    }
    if (!unreadMessages.isEmpty()) {
        Message[] unseenMessages = new Message[unreadMessages.size()];
        unreadMessages.toArray(unseenMessages);
        try {
            emailFolder.setFlags(unseenMessages, new Flags(Flags.Flag.SEEN), false);
        } catch (MessagingException e) {
            if (!store.isConnected()) {
                store.connect();
                emailFolder.setFlags(unseenMessages, new Flags(Flags.Flag.SEEN), false);
                // anyway we need to push this update.
                flushUnseenMessages = unseenMessages;
            } else {
                // anyway we need to push this update.
                flushUnseenMessages = unseenMessages;
            }
        }
    }
}

Example 5 with TaskContext

use of org.apache.airavata.gfac.core.context.TaskContext in project airavata by apache.

the class EmailBasedMonitor method stopMonitor.

@Override
public void stopMonitor(String jobId, boolean runOutflow) {
    TaskContext taskContext = jobMonitorMap.remove(jobId);
    if (taskContext != null && runOutflow) {
        try {
            ProcessContext pc = taskContext.getParentProcessContext();
            if (taskContext.isCancel()) {
                // Moved job status to cancel
                JobModel jobModel = pc.getJobModel();
                JobStatus newJobStatus = new JobStatus(JobState.CANCELED);
                newJobStatus.setReason("Moving job status to cancel, as we didn't see any email from this job " + "for a while after execute job cancel command. This may happen if job was in queued state " + "when we run the cancel command");
                jobModel.setJobStatuses(Arrays.asList(newJobStatus));
                GFacUtils.saveJobStatus(pc, jobModel);
            }
            ProcessStatus pStatus = new ProcessStatus(ProcessState.CANCELLING);
            pStatus.setReason("Job cancelled");
            pc.setProcessStatus(pStatus);
            GFacUtils.saveAndPublishProcessStatus(pc);
            GFacThreadPoolExecutor.getCachedThreadPool().execute(new GFacWorker(pc));
        } catch (GFacException e) {
            log.info("[EJM]: Error while running output tasks", e);
        }
    }
}