Examples with Stage pipelite.stage.Stage used on opensource projects

Example 41 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class SimpleLsfExecutorSubmitCmdTest method cmdNoMemUnitsNoDuration.

@Test
public void cmdNoMemUnitsNoDuration() throws IOException {
    SimpleLsfExecutor executor = new SimpleLsfExecutor();
    executor.setCmd("test");
    SimpleLsfExecutorParameters params = SimpleLsfExecutorParameters.builder().logDir(Files.createTempDirectory("TEMP").toString()).cpu(2).memory(1).queue("TEST").timeout(Duration.ofMinutes(1)).build();
    executor.setExecutorParams(params);
    Stage stage = Stage.builder().stageName(STAGE_NAME).executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName(PIPELINE_NAME).processId(PROCESS_ID).stage(stage).build();
    String logDir = "\"" + params.resolveLogDir(request, LsfFilePathResolver.Format.WITH_LSF_PATTERN) + "\"";
    String logFileName = params.resolveLogFileName(request);
    executor.setOutFile(params.resolveLogDir(request, LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN));
    String submitCmd = executor.getSubmitCmd(request);
    assertThat(submitCmd).isEqualTo("bsub" + " -outdir " + logDir + " -cwd " + logDir + " -oo " + logFileName + " -n 2 -M 1 -R \"rusage[mem=1]\" -W 1 -q TEST test");
}

Example 42 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class SimpleLsfExecutorSubmitCmdTest method cmdJobGroup.

@Test
public void cmdJobGroup() throws IOException {
    SimpleLsfExecutor executor = new SimpleLsfExecutor();
    executor.setCmd("test");
    SimpleLsfExecutorParameters params = SimpleLsfExecutorParameters.builder().logDir(Files.createTempDirectory("TEMP").toString()).cpu(2).jobGroup("testGroup").timeout(Duration.ofMinutes(1)).queue("TEST").build();
    executor.setExecutorParams(params);
    Stage stage = Stage.builder().stageName(STAGE_NAME).executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName(PIPELINE_NAME).processId(PROCESS_ID).stage(stage).build();
    String logDir = "\"" + params.resolveLogDir(request, LsfFilePathResolver.Format.WITH_LSF_PATTERN) + "\"";
    String logFileName = params.resolveLogFileName(request);
    executor.setOutFile(params.resolveLogDir(request, LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN));
    ;
    String submitCmd = executor.getSubmitCmd(request);
    assertThat(submitCmd).isEqualTo("bsub" + " -outdir " + logDir + " -cwd " + logDir + " -oo " + logFileName + " -n 2 -W 1 -g testGroup -q TEST test");
}

Example 43 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class SshCmdExecutorTest method test.

@Test
@EnabledIfEnvironmentVariable(named = "PIPELITE_TEST_SSH_HOST", matches = ".+")
public void test() {
    String stageName = PipeliteIdCreator.stageName();
    CmdExecutor<CmdExecutorParameters> executor = StageExecutor.createCmdExecutor("echo test");
    executor.setExecutorParams(CmdExecutorParameters.builder().host(sshTestConfiguration.getHost()).user(sshTestConfiguration.getUser()).timeout(Duration.ofSeconds(30)).build());
    Stage stage = Stage.builder().stageName(stageName).executor(executor).build();
    stage.execute(PIPELINE_NAME, PROCESS_ID, (result) -> {
        // Ignore timeout errors.
        if (result.isErrorType(ErrorType.TIMEOUT_ERROR)) {
            return;
        }
        assertThat(result.isSuccess()).isTrue();
        assertThat(result.getAttribute(StageExecutorResultAttribute.COMMAND)).isEqualTo("echo test");
        assertThat(result.getAttribute(StageExecutorResultAttribute.EXIT_CODE)).isEqualTo("0");
        assertThat(result.getStageLog()).contains("test\n");
    });
}

Example 44 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class DependencyResolver method getDependentStages.

private static void getDependentStages(Process process, Stage stage, boolean include, Set<Stage> dependentStages) {
    process.getStageGraph().edgesOf(stage).forEach(e -> {
        Stage edgeSource = process.getStageGraph().getEdgeSource(e);
        Stage edgeTarget = process.getStageGraph().getEdgeTarget(e);
        if (edgeSource.equals(stage)) {
            dependentStages.add(edgeTarget);
            getDependentStages(process, edgeTarget, true, dependentStages);
        }
    });
    if (include) {
        dependentStages.add(stage);
    }
}

Example 45 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class DependencyResolver method getDependsOnStagesDirectly.

/**
 * Returns the list of stages that the stage depends on directly.
 *
 * @param process the process
 * @param stage the stage of interest
 * @return the list of stages that the stage depends on directly
 */
public static Set<Stage> getDependsOnStagesDirectly(Process process, Stage stage) {
    Set<Stage> dependsOnStages = new HashSet<>();
    process.getStageGraph().edgesOf(stage).forEach(e -> {
        Stage edgeSource = process.getStageGraph().getEdgeSource(e);
        Stage edgeTarget = process.getStageGraph().getEdgeTarget(e);
        if (edgeTarget.equals(stage)) {
            dependsOnStages.add(edgeSource);
        }
    });
    return dependsOnStages;
}