Examples with PipelineTask com.epam.pipeline.entity.pipeline.PipelineTask used on opensource projects

Example 1 with PipelineTask

use of com.epam.pipeline.entity.pipeline.PipelineTask in project cloud-pipeline by epam.

the class RunLogManager method saveLog.

@Transactional(propagation = Propagation.REQUIRED)
public RunLog saveLog(final RunLog runLog) {
    Assert.notNull(runLog.getRunId(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, "runId", RunLog.class.getSimpleName()));
    Assert.notNull(runLog.getDate(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, "date", RunLog.class.getSimpleName()));
    Assert.notNull(runLog.getStatus(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, "status", RunLog.class.getSimpleName()));
    PipelineRun run = pipelineRunDao.loadPipelineRun(runLog.getRunId());
    Assert.notNull(run, messageHelper.getMessage(MessageConstants.ERROR_PIPELINE_NOT_FOUND, runLog.getRunId()));
    if (!StringUtils.isEmpty(runLog.getLogText())) {
        runLog.setLogText(runLog.getLogText().replaceAll("\\u0000", ""));
    }
    // Check previous status, it may differ from pod status as error may occur during
    // results upload to s3
    TaskStatus statusToSave = runLog.getStatus();
    if (!StringUtils.isEmpty(runLog.getTaskName())) {
        PipelineTask task = self.loadPreviousTaskStatus(run, runLog);
        if (task != null && task.getStatus().isFinal()) {
            statusToSave = task.getStatus();
        }
    }
    // this status
    if (!statusToSave.isFinal() && run.getStatus().isFinal()) {
        statusToSave = run.getStatus();
    }
    runLog.setStatus(statusToSave);
    runLogDao.createRunLog(runLog);
    return runLog;
}

Example 2 with PipelineTask

use of com.epam.pipeline.entity.pipeline.PipelineTask in project cloud-pipeline by epam.

the class RunLogManager method getPodLogs.

private List<RunLog> getPodLogs(PipelineRun run) {
    String logText = StringUtils.isBlank(run.getPodIP()) ? "Node initialization in progress." : kubernetesManager.getPodLogs(run.getPodId());
    RunLog log = new RunLog();
    log.setRunId(run.getId());
    log.setStatus(TaskStatus.RUNNING);
    log.setDate(run.getCreatedDate());
    log.setTask(new PipelineTask(consoleLogTask));
    log.setLogText(logText);
    return Collections.singletonList(log);
}

Example 3 with PipelineTask

use of com.epam.pipeline.entity.pipeline.PipelineTask in project cloud-pipeline by epam.

the class GraphReader method createGraphFromScriptOutput.

protected TaskGraphVO createGraphFromScriptOutput(String output) {
    String[] lines = output.split("\\n");
    if (lines.length == 0) {
        return new TaskGraphVO(new TaskNode(new PipelineTask()));
    }
    // first line is always name of the root task
    String mainTaskName = lines[0].trim();
    // important: for main task inputs and outputs are reverted, due
    // to luigi's approach to graph calculation
    TaskNode mainTask = new TaskNode(new PipelineTask(mainTaskName));
    TaskNode currentTask = mainTask;
    Map<String, TaskNode> tasks = new HashMap<>();
    tasks.put(mainTaskName, mainTask);
    mainTask.getTask().setId(1L);
    for (int i = 1; i < lines.length; i++) {
        String line = lines[i];
        if (line.isEmpty()) {
            continue;
        }
        if (line.startsWith(INPUT_START) && currentTask != null) {
            String value = line.substring(INPUT_START.length());
            parseValue(currentTask.getInputs(), value);
        } else if (line.startsWith(OUTPUT_START) && currentTask != null) {
            String value = line.substring(OUTPUT_START.length());
            parseValue(currentTask.getOutputs(), value);
        } else if (line.startsWith(TOOL_START) && currentTask != null) {
            parseTool(currentTask, line);
        } else if (line.contains(DEPENDENCY_DELIMITER)) {
            String[] dependency = line.split(DEPENDENCY_DELIMITER);
            TaskNode fromTask = getOrCreateNode(dependency[0].trim(), tasks);
            String secondTaskName = dependency.length > 1 ? dependency[1].trim() : "";
            // input to all pipeline
            if (dependency.length == 1 || secondTaskName.isEmpty()) {
                currentTask = null;
                mainTask.getOutputs().addAll(fromTask.getInputs());
            } else {
                TaskNode toTask = getOrCreateNode(secondTaskName, tasks);
                currentTask = toTask;
                toTask.addParent(fromTask.getTask().getId());
            }
        }
    }
    // switch output and input for main task
    mainTask.switchInputOutput();
    TaskGraphVO graph = new TaskGraphVO(mainTask);
    graph.setTasks(new ArrayList<>(tasks.values()));
    return graph;
}

Example 4 with PipelineTask

use of com.epam.pipeline.entity.pipeline.PipelineTask in project cloud-pipeline by epam.

the class PodMonitor method checkChildrenPods.

private boolean checkChildrenPods(PipelineRun run, KubernetesClient client, Pod parent) {
    try {
        // check all children
        List<PipelineTask> tasks = runLogManager.loadTasksByRunId(run.getId());
        tasks.forEach(task -> {
            if (task.getStatus() == TaskStatus.RUNNING && !StringUtils.isEmpty(task.getInstance())) {
                Pod pod = client.pods().inNamespace(kubeNamespace).withName(task.getInstance()).get();
                getPodLogs(run, client, pod);
            }
        });
        if (run.getNodeCount() != null && run.getNodeCount() > 0) {
            clearWorkerNodes(run, client);
        }
        // save luigi log
        getPodLogs(run, client, parent);
        // delete all pods
        LOGGER.debug("Clearing pods for successful pipeline: {}.", run.getPodId());
        client.pods().inNamespace(kubeNamespace).withLabel(PIPELINE_ID_LABEL, run.getPodId()).delete();
    } catch (KubernetesClientException e) {
        LOGGER.debug(e.getMessage(), e);
    }
    return ensurePipelineIsDeleted(String.valueOf(run.getId()), run.getPodId(), client);
}

Example 5 with PipelineTask

use of com.epam.pipeline.entity.pipeline.PipelineTask in project cloud-pipeline by epam.

the class RunLogManagerTest method downloadLogs.

@Test
public void downloadLogs() throws Exception {
    PipelineRun run = new PipelineRun(1L, "");
    List<RunLog> logs = new ArrayList<>();
    logs.add(RunLog.builder().date(Date.from(Instant.now())).task(new PipelineTask(FIRST_TASK)).logText("First task Log1").build());
    logs.add(RunLog.builder().date(Date.from(Instant.now())).task(new PipelineTask(SECOND_TASK)).logText("Second task Log1").build());
    logs.add(RunLog.builder().date(Date.from(Instant.now())).task(new PipelineTask(FIRST_TASK)).logText("First task Log2").build());
    Mockito.when(runManagerMock.loadPipelineRun(run.getId())).thenReturn(run);
    Mockito.when(logDao.loadAllLogsForRun(run.getId())).thenReturn(logs);
    String result = logManager.downloadLogs(run);
    Assert.assertNotNull(result);
    Assert.assertTrue(!result.isEmpty());
}