Examples with Stage pipelite.stage.Stage used on opensource projects

Example 51 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class TestType method spyStageService.

public static void spyStageService(StageService stageServiceSpy) {
    if (!Mockito.mockingDetails(stageServiceSpy).isSpy()) {
        throw new RuntimeException("StageService must be a spy");
    }
    // Last and next exit code
    doAnswer(invocation -> {
        StageEntity stageEntity = (StageEntity) invocation.callRealMethod();
        AtomicReference<TestType> testTypeRef = new AtomicReference<>();
        handleErrors(testTypeRef, () -> {
            Stage stage = invocation.getArgument(0);
            StageExecutorResult result = invocation.getArgument(1);
            StageMapKey key = registeredKey(stage);
            TestType testType = testType(key);
            testTypeRef.set(testType);
            String lastExitCode = result.getAttribute(StageExecutorResultAttribute.EXIT_CODE);
            if (lastExitCode == null) {
                testType.failedAsserts.add("Stage has no exit code");
            } else if (!lastExitCode.equals(stageNextExitCode.get(key))) {
                testType.failedAsserts.add("Unexpected stage exit code " + lastExitCode + " and not " + stageNextExitCode.get(key));
            }
            setLastExitCode(lastExitCode, key);
            setNextExitCode(testType, stage, key);
        });
        return stageEntity;
    }).when(stageServiceSpy).endExecution(any(), any());
    // Stage deserialization
    doAnswer(invocation -> {
        Stage stage = invocation.getArgument(0);
        StageEntity savedStageEntity = (StageEntity) invocation.callRealMethod();
        AtomicReference<TestType> testTypeRef = new AtomicReference<>();
        handleErrors(testTypeRef, () -> {
            StageMapKey key = registeredKey(stage);
            TestType testType = testType(key);
            testTypeRef.set(testType);
            if (!stage.getStageEntity().equals(savedStageEntity)) {
                testType.failedAsserts.add("Stage entity " + stage.getStageEntity().toString() + " is different from saved stage entity: " + savedStageEntity.toString());
            }
        });
        return null;
    }).when(stageServiceSpy).saveStage(any());
}

Example 52 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class RetryServiceTest method retryFailedSchedule.

@Test
public void retryFailedSchedule() {
    String serviceName = PipeliteIdCreator.serviceName();
    String processId = PipeliteIdCreator.processId();
    RegisteredPipeline registeredPipeline = registeredPipelineService.getRegisteredPipeline(SCHEDULE_NAME);
    Process process = ProcessFactory.create(processId, registeredPipeline);
    // Failed process
    process.setProcessEntity(processService.createExecution(SCHEDULE_NAME, processId, 1));
    processService.startExecution(process.getProcessEntity());
    ProcessEntity processEntity = processService.endExecution(process, ProcessState.FAILED);
    // Failed stage
    Stage stage = process.getStage(STAGE_NAME).get();
    stageService.createExecution(SCHEDULE_NAME, processId, stage);
    stageService.startExecution(stage);
    stageService.endExecution(stage, StageExecutorResult.error());
    StageEntity stageEntity = stage.getStageEntity();
    assertThat(processEntity.getProcessState()).isEqualTo(ProcessState.FAILED);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ERROR);
    // Failed schedule
    scheduleService.createSchedule(serviceName, SCHEDULE_NAME, CRON);
    ScheduleEntity scheduleEntity = scheduleService.startExecution(SCHEDULE_NAME, processId);
    ZonedDateTime nextTime = CronUtils.launchTime(CRON, ZonedDateTime.now().plusHours(1));
    scheduleEntity = scheduleService.endExecution(processEntity, nextTime);
    assertThat(scheduleEntity.isFailed()).isTrue();
    // Create schedule runner
    processRunnerPoolManager._createScheduleRunner();
    ScheduleRunner scheduleRunner = runnerService.getScheduleRunner();
    scheduleRunner.setMaximumExecutions(SCHEDULE_NAME, 1);
    assertThat(scheduleRunner.getScheduleCrons().size()).isOne();
    assertThat(scheduleRunner.getScheduleCrons().get(0).getPipelineName()).isEqualTo(SCHEDULE_NAME);
    // Check schedule state
    assertSetupSchedule(serviceName, SCHEDULE_NAME, processId, CRON, nextTime);
    ZonedDateTime retryTime = ZonedDateTime.now();
    Time.wait(Duration.ofMillis(1000));
    scheduleRunner.startUp();
    // Retry
    retryService.retry(SCHEDULE_NAME, processId);
    while (!scheduleRunner.isIdle()) {
        scheduleRunner.runOneIteration();
        Time.wait(Duration.ofMillis(100));
    }
    // Check schedule state
    assertRetriedSchedule(serviceName, SCHEDULE_NAME, processId, CRON, retryTime);
    // Check process state
    processEntity = processService.getSavedProcess(SCHEDULE_NAME, processId).get();
    assertThat(processEntity.getPipelineName()).isEqualTo(SCHEDULE_NAME);
    assertThat(processEntity.getProcessId()).isEqualTo(processId);
    assertThat(processEntity.getStartTime()).isNotNull();
    assertThat(processEntity.getEndTime()).isNotNull();
    assertThat(processEntity.getProcessState()).isEqualTo(ProcessState.COMPLETED);
    // Check stage state
    stageEntity = stageService.getSavedStage(SCHEDULE_NAME, processId, STAGE_NAME).get();
    assertThat(stageEntity.getPipelineName()).isEqualTo(SCHEDULE_NAME);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(STAGE_NAME);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.SUCCESS);
    assertThat(stageEntity.getStartTime()).isNotNull();
    assertThat(stageEntity.getEndTime()).isNotNull();
}

Example 53 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class DependencyResolverTest method othersActivePendingIndependentOneErrorMaxRetriesLeft.

@Test
public void othersActivePendingIndependentOneErrorMaxRetriesLeft() {
    for (StageState stageState : EnumSet.of(ACTIVE, PENDING)) {
        ProcessBuilder builder = createProcessBuilder();
        Process process = builder.execute("STAGE1").withSyncTestExecutor(StageExecutorState.ERROR, ExecutorParameters.builder().maximumRetries(1).immediateRetries(0).build()).execute("STAGE2").withSyncTestExecutor().execute("STAGE3").withSyncTestExecutor().execute("STAGE4").withSyncTestExecutor().build();
        Set<Stage> stages = process.getStages();
        int stageNumber = 0;
        for (Stage stage : process.getStages()) {
            if (stageNumber == 0) {
                simulateStageExecution(stage, StageState.ERROR);
            } else {
                simulateStageExecution(stage, stageState);
            }
            stageNumber++;
        }
        assertGetDependsOnStages(process, "STAGE1", Collections.emptyList());
        assertGetDependsOnStages(process, "STAGE2", Collections.emptyList());
        assertGetDependsOnStages(process, "STAGE3", Collections.emptyList());
        assertGetDependsOnStages(process, "STAGE4", Collections.emptyList());
        assertGetDependsOnStagesDirectly(process, "STAGE1", Collections.emptyList());
        assertGetDependsOnStagesDirectly(process, "STAGE2", Collections.emptyList());
        assertGetDependsOnStagesDirectly(process, "STAGE3", Collections.emptyList());
        assertGetDependsOnStagesDirectly(process, "STAGE4", Collections.emptyList());
        assertGetDependentStages(process, "STAGE1", Collections.emptyList());
        assertGetDependentStages(process, "STAGE2", Collections.emptyList());
        assertGetDependentStages(process, "STAGE3", Collections.emptyList());
        assertGetDependentStages(process, "STAGE4", Collections.emptyList());
        assertGetPermanentFailedStages(stages, Collections.emptyList());
        assertGetImmediatelyExecutableStages(process, Arrays.asList("STAGE2", "STAGE3", "STAGE4"));
        assertGetEventuallyExecutableStages(process, Arrays.asList("STAGE1", "STAGE2", "STAGE3", "STAGE4"));
    }
}

Example 54 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class DependencyResolverTest method allActivePendingDependOnPrevious.

@Test
public void allActivePendingDependOnPrevious() {
    for (StageState stageState : EnumSet.of(ACTIVE, PENDING)) {
        ProcessBuilder builder = createProcessBuilder();
        Process process = builder.execute("STAGE1").withSyncTestExecutor().executeAfterPrevious("STAGE2").withSyncTestExecutor().executeAfterPrevious("STAGE3").withSyncTestExecutor().build();
        Set<Stage> stages = process.getStages();
        for (Stage stage : process.getStages()) {
            simulateStageExecution(stage, stageState);
        }
        assertGetDependsOnStages(process, "STAGE1", Collections.emptyList());
        assertGetDependsOnStages(process, "STAGE2", Arrays.asList("STAGE1"));
        assertGetDependsOnStages(process, "STAGE3", Arrays.asList("STAGE1", "STAGE2"));
        assertGetDependsOnStagesDirectly(process, "STAGE1", Collections.emptyList());
        assertGetDependsOnStagesDirectly(process, "STAGE2", Arrays.asList("STAGE1"));
        assertGetDependsOnStagesDirectly(process, "STAGE3", Arrays.asList("STAGE2"));
        assertGetDependentStages(process, "STAGE1", Arrays.asList("STAGE2", "STAGE3"));
        assertGetDependentStages(process, "STAGE2", Arrays.asList("STAGE3"));
        assertGetDependentStages(process, "STAGE3", Collections.emptyList());
        assertGetPermanentFailedStages(stages, Collections.emptyList());
        assertGetImmediatelyExecutableStages(process, Arrays.asList("STAGE1"));
        assertGetEventuallyExecutableStages(process, Arrays.asList("STAGE1", "STAGE2", "STAGE3"));
    }
}

Example 55 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class DependencyResolverTest method othersActivePendingDependOnFirstInErrorNoRetriesLeft.

@Test
public void othersActivePendingDependOnFirstInErrorNoRetriesLeft() {
    for (StageState stageState : EnumSet.of(ACTIVE, PENDING)) {
        ProcessBuilder builder = createProcessBuilder();
        Process process = builder.execute("STAGE1").withSyncTestExecutor(StageExecutorState.ERROR, NO_RETRIES_EXECUTOR_PARAMS).executeAfter("STAGE2", "STAGE1").withSyncTestExecutor().executeAfter("STAGE3", "STAGE1").withSyncTestExecutor().build();
        Set<Stage> stages = process.getStages();
        int stageNumber = 0;
        for (Stage stage : process.getStages()) {
            if (stageNumber == 0) {
                simulateStageExecution(stage, StageState.ERROR);
            } else {
                simulateStageExecution(stage, stageState);
            }
            stageNumber++;
        }
        assertGetDependsOnStages(process, "STAGE1", Collections.emptyList());
        assertGetDependsOnStages(process, "STAGE2", Arrays.asList("STAGE1"));
        assertGetDependsOnStages(process, "STAGE3", Arrays.asList("STAGE1"));
        assertGetDependsOnStagesDirectly(process, "STAGE1", Collections.emptyList());
        assertGetDependsOnStagesDirectly(process, "STAGE2", Arrays.asList("STAGE1"));
        assertGetDependsOnStagesDirectly(process, "STAGE3", Arrays.asList("STAGE1"));
        assertGetDependentStages(process, "STAGE1", Arrays.asList("STAGE2", "STAGE3"));
        assertGetDependentStages(process, "STAGE2", Collections.emptyList());
        assertGetDependentStages(process, "STAGE3", Collections.emptyList());
        assertGetPermanentFailedStages(stages, Arrays.asList("STAGE1"));
        assertGetImmediatelyExecutableStages(process, Collections.emptyList());
        assertGetEventuallyExecutableStages(process, Collections.emptyList());
    }
}