Examples with StageExecutorResult pipelite.stage.executor.StageExecutorResult used on opensource projects

Example 11 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class DescribeJobsTest method success.

@Test
public void success() {
    int requestLimit = 10;
    int requestCnt = 100;
    AtomicInteger actualDescribeJobsCallCnt = new AtomicInteger();
    AtomicInteger actualRequestCnt = new AtomicInteger();
    DescribeJobsCallback<Integer, String> describeJobsCallback = (requestList, context) -> {
        actualDescribeJobsCallCnt.incrementAndGet();
        actualRequestCnt.addAndGet(requestList.size());
        return requestList.stream().collect(Collectors.toMap(i -> i, i -> StageExecutorResult.success()));
    };
    String executorContext = "test";
    final DescribeJobs<Integer, String> describeJobs = new DescribeJobs(mock(ServiceConfiguration.class), mock(InternalErrorService.class), requestLimit, executorContext, describeJobsCallback);
    IntStream.range(0, requestCnt).forEach(i -> describeJobs.addRequest(i));
    assertThat(describeJobs.getActiveRequests().size()).isEqualTo(requestCnt);
    describeJobs.makeRequests();
    assertThat(actualDescribeJobsCallCnt.get()).isEqualTo(requestCnt / requestLimit);
    assertThat(actualRequestCnt.get()).isEqualTo(requestCnt);
    // Check that there are no active requests.
    assertThat(describeJobs.getActiveRequests().size()).isEqualTo(0);
    // Check that the results are correct.
    IntStream.range(0, requestCnt).forEach(i -> {
        StageExecutorResult result = describeJobs.getResult(i, null);
        assertThat(result.isSuccess()).isTrue();
    });
    // Check that the requests have been removed.
    IntStream.range(0, requestCnt).forEach(i -> assertThat(describeJobs.isRequest(i)).isFalse());
}

Example 12 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class DescribeJobsTest method active.

@Test
public void active() {
    int requestLimit = 10;
    int requestCnt = 100;
    AtomicInteger actualDescribeJobsCallCnt = new AtomicInteger();
    AtomicInteger actualRequestCnt = new AtomicInteger();
    DescribeJobsCallback<Integer, String> describeJobsCallback = (requestList, context) -> {
        actualDescribeJobsCallCnt.incrementAndGet();
        actualRequestCnt.addAndGet(requestList.size());
        return requestList.stream().collect(Collectors.toMap(i -> i, i -> StageExecutorResult.active()));
    };
    String executorContext = "test";
    final DescribeJobs<Integer, String> describeJobs = new DescribeJobs(mock(ServiceConfiguration.class), mock(InternalErrorService.class), requestLimit, executorContext, describeJobsCallback);
    IntStream.range(0, requestCnt).forEach(i -> describeJobs.addRequest(i));
    assertThat(describeJobs.getActiveRequests().size()).isEqualTo(requestCnt);
    describeJobs.makeRequests();
    assertThat(actualDescribeJobsCallCnt.get()).isEqualTo(requestCnt / requestLimit);
    assertThat(actualRequestCnt.get()).isEqualTo(requestCnt);
    // Check that the requests remain active.
    assertThat(describeJobs.getActiveRequests().size()).isEqualTo(requestCnt);
    // Check that the results are correct.
    IntStream.range(0, requestCnt).forEach(i -> {
        StageExecutorResult result = describeJobs.getResult(i, null);
        assertThat(result.isActive()).isTrue();
    });
    // Check that the requests have not been removed.
    IntStream.range(0, requestCnt).forEach(i -> assertThat(describeJobs.isRequest(i)).isTrue());
}

Example 13 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class AbstractLsfExecutorExtractTest method extractResultFromOutFileError.

@Test
public void extractResultFromOutFileError() {
    StageExecutorResult result = AbstractLsfExecutor.extractResultFromBhistOutputOrOutFile("\n" + "\n" + "Job <dfsddf> was submitted from host <noah-login-02> by user <rasko> in cluster <EBI> at Sun Jan 24 18:06:13 2021\n" + "Job was executed on host(s) <hx-noah-24-04>, in queue <research-rh74>, as user <rasko> in cluster <EBI> at Sun Jan 24 18:06:14 2021\n" + "</homes/rasko> was used as the home directory.\n" + "</homes/rasko> was used as the working directory.\n" + "Started at Sun Jan 24 18:06:14 2021\n" + "Terminated at Sun Jan 24 18:06:14 2021\n" + "Results reported at Sun Jan 24 18:06:14 2021\n" + "\n" + "Your job looked like:\n" + "\n" + "------------------------------------------------------------\n" + "# LSBATCH: User input\n" + "dfsddf\n" + "------------------------------------------------------------\n" + "\n" + "Exited with exit code 127.\n" + "\n" + "Resource usage summary:\n" + "\n" + "    CPU time :                                   0.03 sec.\n" + "    Max Memory :                                 -\n" + "    Average Memory :                             -\n" + "    Total Requested Memory :                     -\n" + "    Delta Memory :                               -\n" + "    Max Swap :                                   -\n" + "    Max Processes :                              -\n" + "    Max Threads :                                -\n" + "    Run time :                                   0 sec.\n" + "    Turnaround time :                            1 sec.\n" + "\n" + "The output (if any) follows:\n" + "\n" + "/ebi/lsf/ebi-spool2/01/1611511573.6138156: line 8: dfsddf: command not found\n" + "\n");
    assertThat(result.isError()).isTrue();
}

Example 14 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class StageEntityTest method lifecycle.

@Test
public void lifecycle() {
    String pipelineName = PipeliteIdCreator.pipelineName();
    String processId = PipeliteIdCreator.processId();
    String stageName = PipeliteIdCreator.stageName();
    TestExecutor executor = new TestExecutor();
    executor.setExecutorParams(ExecutorParameters.builder().timeout(Duration.ofSeconds(0)).immediateRetries(0).maximumRetries(0).build());
    Stage stage = Stage.builder().stageName(stageName).executor(executor).build();
    // Create execution.
    StageEntity stageEntity = StageEntity.createExecution(pipelineName, processId, stage);
    StageService.prepareSaveStage(stage);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(0);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.PENDING);
    assertThat(stageEntity.getErrorType()).isNull();
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.entity.StageEntityTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"test\" : \"TEST_EXECUTOR_DATA\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 0,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    // Start first execution.
    stageEntity.startExecution();
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(0);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ACTIVE);
    assertThat(stageEntity.getErrorType()).isNull();
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.entity.StageEntityTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"test\" : \"TEST_EXECUTOR_DATA\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 0,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    // End first execution.
    StageExecutorResult firstExecutionResult = StageExecutorResult.error();
    stageEntity.endExecution(firstExecutionResult);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(1);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ERROR);
    assertThat(stageEntity.getErrorType()).isEqualTo(ErrorType.EXECUTION_ERROR);
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNotNull();
    assertThat(stageEntity.getStartTime()).isBeforeOrEqualTo(stageEntity.getEndTime());
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.entity.StageEntityTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"test\" : \"TEST_EXECUTOR_DATA\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 0,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    // Start second execution.
    stageEntity.startExecution();
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(1);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ACTIVE);
    assertThat(stageEntity.getErrorType()).isNull();
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.entity.StageEntityTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"test\" : \"TEST_EXECUTOR_DATA\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 0,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    // End second execution.
    StageExecutorResult secondExecutionResult = StageExecutorResult.success();
    stageEntity.endExecution(secondExecutionResult);
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(2);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.SUCCESS);
    assertThat(stageEntity.getErrorType()).isNull();
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNotNull();
    assertThat(stageEntity.getStartTime()).isBeforeOrEqualTo(stageEntity.getEndTime());
    assertThat(stageEntity.getExecutorName()).isEqualTo("pipelite.entity.StageEntityTest$TestExecutor");
    assertThat(stageEntity.getExecutorData()).isEqualTo("{\n" + "  \"test\" : \"TEST_EXECUTOR_DATA\"\n" + "}");
    assertThat(stageEntity.getExecutorParams()).isEqualTo("{\n" + "  \"timeout\" : 0,\n" + "  \"maximumRetries\" : 0,\n" + "  \"immediateRetries\" : 0,\n" + "  \"logLines\" : 1000\n" + "}");
    // Reset execution.
    stageEntity.resetExecution();
    assertThat(stageEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageEntity.getExecutionCount()).isEqualTo(0);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.PENDING);
    assertThat(stageEntity.getErrorType()).isNull();
    assertThat(stageEntity.getResultParams()).isNull();
    assertThat(stageEntity.getStartTime()).isNull();
    assertThat(stageEntity.getEndTime()).isNull();
    assertThat(stageEntity.getExecutorName()).isNull();
    assertThat(stageEntity.getExecutorData()).isNull();
    assertThat(stageEntity.getExecutorParams()).isNull();
}

Example 15 with StageExecutorResult

use of pipelite.stage.executor.StageExecutorResult in project pipelite by enasequence.

the class StageLogEntityTest method lifecycle.

@Test
public void lifecycle() {
    String pipelineName = PipeliteIdCreator.pipelineName();
    String processId = PipeliteIdCreator.processId();
    String stageName = PipeliteIdCreator.stageName();
    Stage stage = Stage.builder().stageName(stageName).executor(StageExecutor.createSyncTestExecutor(StageExecutorState.SUCCESS, null)).build();
    // Create execution.
    StageEntity.createExecution(pipelineName, processId, stage);
    // End execution.
    StageExecutorResult result = StageExecutorResult.error();
    result.setStageLog("TEST3");
    StageLogEntity stageLogEntity = StageLogEntity.endExecution(stage.getStageEntity(), result);
    assertThat(stageLogEntity.getPipelineName()).isEqualTo(pipelineName);
    assertThat(stageLogEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageLogEntity.getStageName()).isEqualTo(stageName);
    assertThat(stageLogEntity.getStageLog()).isEqualTo("TEST3");
}