Examples with Stage pipelite.stage.Stage used on opensource projects

Example 21 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class RetryServiceTest method retryFailedProcess.

@Test
public void retryFailedProcess() {
    String processId = PipeliteIdCreator.processId();
    RegisteredPipeline registeredPipeline = registeredPipelineService.getRegisteredPipeline(PIPELINE_NAME);
    Process process = ProcessFactory.create(processId, registeredPipeline);
    // Failed process
    process.setProcessEntity(processService.createExecution(PIPELINE_NAME, processId, 1));
    processService.startExecution(process.getProcessEntity());
    ProcessEntity processEntity = processService.endExecution(process, ProcessState.FAILED);
    // Failed stage
    Stage stage = process.getStage(STAGE_NAME).get();
    stageService.createExecution(PIPELINE_NAME, processId, stage);
    stageService.startExecution(stage);
    stageService.endExecution(stage, StageExecutorResult.error());
    StageEntity stageEntity = stage.getStageEntity();
    assertThat(processEntity.getProcessState()).isEqualTo(ProcessState.FAILED);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.ERROR);
    // Retry
    retryService.retry(PIPELINE_NAME, processId);
    // Check process state
    processEntity = processService.getSavedProcess(PIPELINE_NAME, processId).get();
    assertThat(processEntity.getPipelineName()).isEqualTo(PIPELINE_NAME);
    assertThat(processEntity.getProcessId()).isEqualTo(processId);
    assertThat(processEntity.getStartTime()).isNotNull();
    assertThat(processEntity.getEndTime()).isNull();
    assertThat(processEntity.getProcessState()).isEqualTo(ProcessState.ACTIVE);
    // Check stage state
    stageEntity = stageService.getSavedStage(PIPELINE_NAME, processId, STAGE_NAME).get();
    assertThat(stageEntity.getPipelineName()).isEqualTo(PIPELINE_NAME);
    assertThat(stageEntity.getProcessId()).isEqualTo(processId);
    assertThat(stageEntity.getStageName()).isEqualTo(STAGE_NAME);
    assertThat(stageEntity.getStageState()).isEqualTo(StageState.PENDING);
    assertThat(stageEntity.getStartTime()).isNull();
    assertThat(stageEntity.getEndTime()).isNull();
}

Example 22 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class LocalCmdExecutorTest method test.

@Test
public void test() {
    String stageName = PipeliteIdCreator.stageName();
    CmdExecutor<CmdExecutorParameters> executor = StageExecutor.createCmdExecutor("echo test");
    executor.setExecutorParams(CmdExecutorParameters.builder().build());
    Stage stage = Stage.builder().stageName(stageName).executor(executor).build();
    stage.execute(PIPELINE_NAME, PROCESS_ID, (result) -> {
        assertThat(result.isSuccess()).isTrue();
        assertThat(result.getAttribute(StageExecutorResultAttribute.COMMAND)).isEqualTo("echo test");
        assertThat(result.getAttribute(StageExecutorResultAttribute.EXIT_CODE)).isEqualTo("0");
        assertThat(result.getStageLog()).contains("test\n");
    });
}

Example 23 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class LsfExecutorSubmitCmdTest method test.

@Test
public void test() throws IOException {
    LsfExecutor executor = new LsfExecutor();
    executor.setCmd("test");
    LsfExecutorParameters params = LsfExecutorParameters.builder().logDir(Files.createTempDirectory("TEMP").toString()).definition("pipelite/executor/lsf.yaml").format(LsfExecutorParameters.Format.YAML).build();
    executor.setExecutorParams(params);
    Stage stage = Stage.builder().stageName(STAGE_NAME).executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName(PIPELINE_NAME).processId(PROCESS_ID).stage(stage).build();
    String definitionFile = params.resolveDefinitionFile(request, LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN);
    String logDir = "\"" + params.resolveLogDir(request, LsfFilePathResolver.Format.WITH_LSF_PATTERN) + "\"";
    String logFileName = params.resolveLogFileName(request);
    executor.setOutFile(params.resolveLogDir(request, LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN));
    executor.setDefinitionFile(definitionFile);
    String submitCmd = executor.getSubmitCmd(request);
    assertThat(submitCmd).isEqualTo("bsub" + " -outdir " + logDir + " -cwd " + logDir + " -oo " + logFileName + " -yaml " + definitionFile + " test");
}

Example 24 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class SimpleLsfExecutorSubmitCmdTest method cmdMemUnitsM.

@Test
public void cmdMemUnitsM() throws IOException {
    SimpleLsfExecutor executor = new SimpleLsfExecutor();
    executor.setCmd("test");
    SimpleLsfExecutorParameters params = SimpleLsfExecutorParameters.builder().logDir(Files.createTempDirectory("TEMP").toString()).cpu(2).memory(1).memoryUnits("M").memoryTimeout(Duration.ofMinutes(1)).queue("TEST").timeout(Duration.ofMinutes(1)).build();
    executor.setExecutorParams(params);
    Stage stage = Stage.builder().stageName(STAGE_NAME).executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName(PIPELINE_NAME).processId(PROCESS_ID).stage(stage).build();
    String logDir = "\"" + params.resolveLogDir(request, LsfFilePathResolver.Format.WITH_LSF_PATTERN) + "\"";
    String logFileName = params.resolveLogFileName(request);
    executor.setOutFile(params.resolveLogDir(request, LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN));
    String submitCmd = executor.getSubmitCmd(request);
    assertThat(submitCmd).isEqualTo("bsub" + " -outdir " + logDir + " -cwd " + logDir + " -oo " + logFileName + " -n 2 -M 1M -R \"rusage[mem=1M:duration=1]\" -W 1 -q TEST test");
}

Example 25 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class DependencyResolver method getDependsOnStages.

private static void getDependsOnStages(Process process, Stage stage, Set<Stage> dependsOnStages) {
    process.getStageGraph().edgesOf(stage).forEach(e -> {
        Stage edgeSource = process.getStageGraph().getEdgeSource(e);
        Stage edgeTarget = process.getStageGraph().getEdgeTarget(e);
        if (edgeTarget.equals(stage)) {
            dependsOnStages.add(edgeSource);
            getDependsOnStages(process, edgeSource, dependsOnStages);
        }
    });
}