Examples with StageExecutorResult pipelite.stage.executor.StageExecutorResult used on opensource projects

Example 1 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class StageServiceTest method lifecycle.

@Test
public void lifecycle() {
    String pipelineName = PipeliteIdCreator.pipelineName();
    String processId = PipeliteIdCreator.processId();
    String stageName = PipeliteIdCreator.stageName();
    TestExecutor executor = new TestExecutor(StageState.SUCCESS);
    executor.setExecutorParams(ExecutorParameters.builder().immediateRetries(0).maximumRetries(1).timeout(Duration.ofSeconds(0)).build());
    Stage stage = Stage.builder().stageName(stageName).executor(executor).build();
    // Create execution.
    service.createExecution(pipelineName, processId, stage);
    StageEntity stageEntity = stage.getStageEntity();
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(0);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.PENDING);
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.service.StageServiceTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"stageState\" : \"SUCCESS\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 1,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    // Start first execution.
    service.startExecution(stage);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(0);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ACTIVE);
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.service.StageServiceTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"stageState\" : \"SUCCESS\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 1,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    assertThat(service.getSavedStage(pipelineName, processId, stageName).get()).isEqualTo(stageEntity);
    assertThat(service.getSavedStageLog(pipelineName, processId, stageName)).isNotPresent();
    // End first execution.
    StageExecutorResult firstExecutionResult = StageExecutorResult.error();
    service.endExecution(stage, firstExecutionResult);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(1);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ERROR);
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNotNull();
    assertThat(stageEntity.getStartTime()).isBeforeOrEqualTo(stageEntity.getEndTime());
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.service.StageServiceTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"stageState\" : \"SUCCESS\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 1,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    assertThat(service.getSavedStage(pipelineName, processId, stageName).get()).isEqualTo(stageEntity);
    Optional<StageLogEntity> stageLogEntity = service.getSavedStageLog(pipelineName, processId, stageName);
    assertThat(stageLogEntity).isPresent();
    // Start second execution.
    service.startExecution(stage);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(1);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ACTIVE);
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.service.StageServiceTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"stageState\" : \"SUCCESS\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 1,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    assertThat(service.getSavedStage(pipelineName, processId, stageName).get()).isEqualTo(stageEntity);
    assertThat(service.getSavedStageLog(pipelineName, processId, stageName)).isNotPresent();
    // End second execution.
    StageExecutorResult secondExecutionResult = StageExecutorResult.success();
    service.endExecution(stage, secondExecutionResult);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(2);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.SUCCESS);
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNotNull();
    assertThat(stageEntity.getStartTime()).isBeforeOrEqualTo(stageEntity.getEndTime());
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.service.StageServiceTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"stageState\" : \"SUCCESS\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 1,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    assertThat(service.getSavedStage(pipelineName, processId, stageName).get()).isEqualTo(stageEntity);
    stageLogEntity = service.getSavedStageLog(pipelineName, processId, stageName);
    assertThat(stageLogEntity).isPresent();
    // Reset execution.
    service.resetExecution(stage);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(0);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.PENDING);
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.service.StageServiceTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"stageState\" : \"SUCCESS\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 1,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    assertThat(service.getSavedStage(pipelineName, processId, stageName).get()).isEqualTo(stageEntity);
    assertThat(service.getSavedStageLog(pipelineName, processId, stageName)).isNotPresent();
}

Example 2 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class KubernetesExecutor method poll.

@Override
protected StageExecutorResult poll(StageExecutorRequest request) {
    String jobId = getJobId();
    logContext(log.atFine(), request).log("Polling Kubernetes job result " + jobId);
    StageExecutorResult result = describeJobs().getResult(jobId, getExecutorParams().getPermanentErrors());
    if (result.isActive()) {
        return StageExecutorResult.active();
    }
    try (KubernetesClient client = kubernetesClient(context)) {
        if (isSaveLogFile(result)) {
            List<Pod> pods = client.pods().inNamespace(namespace).withLabel("job-name", jobId).list().getItems();
            Pod pod = lastPodToStart(pods);
            String log = client.pods().inNamespace(namespace).withName(pod.getMetadata().getName()).tailingLines(getExecutorParams().getLogLines()).getLog();
            result.setStageLog(log);
        }
        terminateJob(client);
    } catch (KubernetesClientException e) {
        throw new PipeliteException("Kubernetes error", e);
    }
    return result;
}

Example 3 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class CmdRunner method result.

/**
 * Returns execution result.
 *
 * @param cmd the command to execute
 * @param exitCode the exit code
 * @param stdout the stdout
 * @param stderr the stderr
 * @return the execution result
 */
static StageExecutorResult result(String cmd, int exitCode, String stdout, String stderr) {
    StageExecutorResult result;
    result = (exitCode == EXIT_CODE_SUCCESS) ? StageExecutorResult.success() : StageExecutorResult.error();
    result.addAttribute(StageExecutorResultAttribute.COMMAND, cmd);
    result.addAttribute(StageExecutorResultAttribute.EXIT_CODE, exitCode);
    result.setStageLog((stdout != null ? stdout : "") + (stderr != null ? stderr : ""));
    return result;
}

Example 4 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class AbstractAsyncExecutor method submit.

private void submit(StageExecutorRequest request, StageExecutorResultCallback resultCallback) {
    submitExecutorService.submit(() -> {
        try {
            ZonedDateTime submitStartTime = ZonedDateTime.now();
            prepareSubmit(request);
            SubmitResult submitResult = submit(request);
            jobId = submitResult.getJobId();
            StageExecutorResult result = submitResult.getResult();
            if (stageMetrics != null) {
                stageMetrics.getAsyncSubmitTimer().record(Duration.between(submitStartTime, ZonedDateTime.now()));
                stageMetrics.endAsyncSubmit();
            }
            if (!result.isError()) {
                if (!result.isSubmitted()) {
                    result = StageExecutorResult.internalError(new PipeliteException("Unexpected state after asynchronous submit: " + result.getExecutorState().name()));
                } else if (jobId == null) {
                    result = StageExecutorResult.internalError(new PipeliteException("Missing job id after asynchronous submit"));
                }
            }
            resultCallback.accept(result);
        } catch (Exception ex) {
            StageExecutorResult result = StageExecutorResult.internalError(ex);
            resultCallback.accept(result);
        } finally {
            submitLock.unlock();
        }
    });
}

Example 5 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class AbstractLsfExecutor method poll.

@Override
protected StageExecutorResult poll(StageExecutorRequest request) {
    String jobId = getJobId();
    logContext(log.atFine(), request).log("Polling LSF job result " + jobId);
    StageExecutorResult result = describeJobs().getResult(describeJobsRequestContext(), getExecutorParams().getPermanentErrors());
    if (result.isActive()) {
        return result;
    }
    ZonedDateTime outFileTimeout = ZonedDateTime.now().plus(getExecutorParams().getLogTimeout());
    if (!isSaveLogFile(result) || readOutFile(request, result, outFileTimeout)) {
        return result;
    }
    return StageExecutorResult.active();
}