Examples with Stage pipelite.stage.Stage used on opensource projects

Example 36 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class AsyncExecutorTestHelper method testExecute.

public static void testExecute(AbstractAsyncExecutor<?, ?> executor, PipeliteExecutorService pipeliteExecutorService, DescribeJobsCacheService describeJobsCacheService, PipeliteMetrics pipeliteMetrics, StageExecutorResultCallback assertAfterSubmit, StageExecutorResultCallback assertAfterPoll) {
    String pipelineName = PipeliteIdCreator.pipelineName();
    String processId = PipeliteIdCreator.processId();
    String stageName = PipeliteIdCreator.stageName();
    Stage stage = Stage.builder().stageName(stageName).executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName(pipelineName).processId(processId).stage(stage).build();
    executor.setSubmitExecutorService(pipeliteExecutorService.submitStage());
    executor.setDescribeJobsService(describeJobsCacheService);
    executor.setStageMetrics(pipeliteMetrics.pipeline(pipelineName).stage());
    AtomicReference<StageExecutorResult> result = new AtomicReference<>();
    executor.execute(request, (r) -> result.set(r));
    while (result.get() == null) {
        Time.wait(Duration.ofSeconds(1));
    }
    assertThat(result.get().isSubmitted()).isTrue();
    assertAfterSubmit.accept(result.get());
    while (!result.get().isSuccess() && !result.get().isError()) {
        executor.execute(request, (r) -> result.set(r));
        Time.wait(Duration.ofSeconds(1));
    }
    // Ignore timeout errors.
    if (result.get().isErrorType(ErrorType.TIMEOUT_ERROR)) {
        return;
    }
    assertAfterPoll.accept(result.get());
}

Example 37 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class LsfExecutorSerializeTest method test.

@Test
public void test() {
    String cmd = "echo test";
    LsfExecutor executor = StageExecutor.createLsfExecutor(cmd);
    Stage stage = Stage.builder().stageName("STAGE_NAME").executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName("PIPELINE_NAME").processId("PROCESS_ID").stage(stage).build();
    LsfExecutorParameters params = LsfExecutorParameters.builder().user("user").logDir("logDir").definitionDir("definitionDir").build();
    LsfFilePathResolver.Format format = LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN;
    executor.setJobId("test");
    executor.setOutFile(new LsfLogFilePathResolver(request, params).getFile(format));
    executor.setDefinitionFile(new LsfDefinitionFilePathResolver(request, params).getFile(format));
    String json = Json.serialize(executor);
    assertThat(json).isEqualTo("{\n" + "  \"jobId\" : \"test\",\n" + "  \"cmd\" : \"echo test\",\n" + "  \"outFile\" : \"logDir/user/PIPELINE_NAME/PROCESS_ID/STAGE_NAME.out\",\n" + "  \"definitionFile\" : \"definitionDir/user/PIPELINE_NAME/PROCESS_ID/STAGE_NAME.job\"\n" + "}");
    LsfExecutor deserializedLsfExecutor = Json.deserialize(json, LsfExecutor.class);
    assertThat(deserializedLsfExecutor.getCmd()).isEqualTo(cmd);
    assertThat(deserializedLsfExecutor.getJobId()).isEqualTo("test");
    assertThat(deserializedLsfExecutor.getOutFile()).isEqualTo("logDir/user/PIPELINE_NAME/PROCESS_ID/STAGE_NAME.out");
    assertThat(deserializedLsfExecutor.getDefinitionFile()).isEqualTo("definitionDir/user/PIPELINE_NAME/PROCESS_ID/STAGE_NAME.job");
}

Example 38 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class SimpleLsfExecutorSerializeTest method test.

@Test
public void test() {
    String cmd = "echo test";
    SimpleLsfExecutor executor = StageExecutor.createSimpleLsfExecutor(cmd);
    Stage stage = Stage.builder().stageName("STAGE_NAME").executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName("PIPELINE_NAME").processId("PROCESS_ID").stage(stage).build();
    SimpleLsfExecutorParameters params = SimpleLsfExecutorParameters.builder().user("user").logDir("logDir").build();
    LsfFilePathResolver.Format format = LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN;
    executor.setJobId("test");
    executor.setOutFile(new LsfLogFilePathResolver(request, params).getFile(format));
    String json = Json.serialize(executor);
    assertThat(json).isEqualTo("{\n" + "  \"jobId\" : \"test\",\n" + "  \"cmd\" : \"echo test\",\n" + "  \"outFile\" : \"logDir/user/PIPELINE_NAME/PROCESS_ID/STAGE_NAME.out\"\n" + "}");
    SimpleLsfExecutor deserializedLsfExecutor = Json.deserialize(json, SimpleLsfExecutor.class);
    assertThat(deserializedLsfExecutor.getCmd()).isEqualTo(cmd);
    assertThat(deserializedLsfExecutor.getJobId()).isEqualTo("test");
    assertThat(deserializedLsfExecutor.getOutFile()).isEqualTo("logDir/user/PIPELINE_NAME/PROCESS_ID/STAGE_NAME.out");
}

Example 39 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class SimpleLsfExecutorSubmitCmdTest method cmdMemUnitsMNoDuration.

@Test
public void cmdMemUnitsMNoDuration() throws IOException {
    SimpleLsfExecutor executor = new SimpleLsfExecutor();
    executor.setCmd("test");
    SimpleLsfExecutorParameters params = SimpleLsfExecutorParameters.builder().logDir(Files.createTempDirectory("TEMP").toString()).cpu(2).memory(1).memoryUnits("M").queue("TEST").timeout(Duration.ofMinutes(1)).build();
    executor.setExecutorParams(params);
    Stage stage = Stage.builder().stageName(STAGE_NAME).executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName(PIPELINE_NAME).processId(PROCESS_ID).stage(stage).build();
    String logDir = "\"" + params.resolveLogDir(request, LsfFilePathResolver.Format.WITH_LSF_PATTERN) + "\"";
    String logFileName = params.resolveLogFileName(request);
    executor.setOutFile(params.resolveLogDir(request, LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN));
    String submitCmd = executor.getSubmitCmd(request);
    assertThat(submitCmd).isEqualTo("bsub" + " -outdir " + logDir + " -cwd " + logDir + " -oo " + logFileName + " -n 2 -M 1M -R \"rusage[mem=1M]\" -W 1 -q TEST test");
}

Example 40 with Stage

use of pipelite.stage.Stage in project pipelite by enasequence.

the class SimpleLsfExecutorSubmitCmdTest method cmdNoMemUnits.

@Test
public void cmdNoMemUnits() throws IOException {
    SimpleLsfExecutor executor = new SimpleLsfExecutor();
    executor.setCmd("test");
    SimpleLsfExecutorParameters params = SimpleLsfExecutorParameters.builder().logDir(Files.createTempDirectory("TEMP").toString()).cpu(2).memory(1).memoryTimeout(Duration.ofMinutes(1)).queue("TEST").timeout(Duration.ofMinutes(1)).build();
    executor.setExecutorParams(params);
    Stage stage = Stage.builder().stageName(STAGE_NAME).executor(executor).build();
    StageExecutorRequest request = StageExecutorRequest.builder().pipelineName(PIPELINE_NAME).processId(PROCESS_ID).stage(stage).build();
    String logDir = "\"" + params.resolveLogDir(request, LsfFilePathResolver.Format.WITH_LSF_PATTERN) + "\"";
    String logFileName = params.resolveLogFileName(request);
    executor.setOutFile(params.resolveLogDir(request, LsfFilePathResolver.Format.WITHOUT_LSF_PATTERN));
    String submitCmd = executor.getSubmitCmd(request);
    assertThat(submitCmd).isEqualTo("bsub" + " -outdir " + logDir + " -cwd " + logDir + " -oo " + logFileName + " -n 2 -M 1 -R \"rusage[mem=1:duration=1]\" -W 1 -q TEST test");
}