Examples with State org.apache.beam.sdk.PipelineResult.State used on opensource projects

Example 6 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class TestDataflowRunner method waitForStreamingJobTermination.

/**
   * Return {@code true} if the job succeeded or {@code false} if it terminated in any other manner.
   */
private boolean waitForStreamingJobTermination(final DataflowPipelineJob job, ErrorMonitorMessagesHandler messageHandler) {
    // In streaming, there are infinite retries, so rather than timeout
    // we try to terminate early by polling and canceling if we see
    // an error message
    options.getExecutorService().submit(new CancelOnError(job, messageHandler));
    // Whether we canceled or not, this gets the final state of the job or times out
    State finalState;
    try {
        finalState = job.waitUntilFinish(Duration.standardSeconds(options.getTestTimeoutSeconds()), messageHandler);
    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (InterruptedException e) {
        Thread.interrupted();
        return false;
    }
    // This cancellation may be the second
    if (finalState == null || !finalState.isTerminal()) {
        LOG.info("Dataflow job {} took longer than {} seconds to complete, cancelling.", job.getJobId(), options.getTestTimeoutSeconds());
        try {
            job.cancel();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return false;
    } else {
        return finalState == State.DONE && !messageHandler.hasSeenError();
    }
}

Example 7 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class DataflowPipelineJobTest method testCumulativeTimeOverflow.

@Test
public void testCumulativeTimeOverflow() throws Exception {
    Dataflow.Projects.Locations.Jobs.Get statusRequest = mock(Dataflow.Projects.Locations.Jobs.Get.class);
    Job statusResponse = new Job();
    statusResponse.setCurrentState("JOB_STATE_RUNNING");
    when(mockJobs.get(eq(PROJECT_ID), eq(REGION_ID), eq(JOB_ID))).thenReturn(statusRequest);
    when(statusRequest.execute()).thenReturn(statusResponse);
    FastNanoClockAndFuzzySleeper clock = new FastNanoClockAndFuzzySleeper();
    DataflowPipelineJob job = new DataflowPipelineJob(DataflowClient.create(options), JOB_ID, options, ImmutableMap.<AppliedPTransform<?, ?, ?>, String>of());
    long startTime = clock.nanoTime();
    State state = job.waitUntilFinish(Duration.millis(4), null, clock, clock);
    assertEquals(null, state);
    long timeDiff = TimeUnit.NANOSECONDS.toMillis(clock.nanoTime() - startTime);
    // Should only have slept for the 4 ms allowed.
    assertThat(timeDiff, lessThanOrEqualTo(4L));
}

Example 8 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class DataflowPipelineJobTest method testWaitToFinishTimeFail.

@Test
public void testWaitToFinishTimeFail() throws Exception {
    Dataflow.Projects.Locations.Jobs.Get statusRequest = mock(Dataflow.Projects.Locations.Jobs.Get.class);
    when(mockJobs.get(eq(PROJECT_ID), eq(REGION_ID), eq(JOB_ID))).thenReturn(statusRequest);
    when(statusRequest.execute()).thenThrow(IOException.class);
    DataflowPipelineJob job = new DataflowPipelineJob(DataflowClient.create(options), JOB_ID, options, ImmutableMap.<AppliedPTransform<?, ?, ?>, String>of());
    long startTime = fastClock.nanoTime();
    State state = job.waitUntilFinish(Duration.millis(4), null, fastClock, fastClock);
    assertEquals(null, state);
    long timeDiff = TimeUnit.NANOSECONDS.toMillis(fastClock.nanoTime() - startTime);
    // Should only have slept for the 4 ms allowed.
    assertEquals(4L, timeDiff);
}

Example 9 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class DataflowPipelineJobTest method testWaitToFinishFail.

@Test
public void testWaitToFinishFail() throws Exception {
    Dataflow.Projects.Locations.Jobs.Get statusRequest = mock(Dataflow.Projects.Locations.Jobs.Get.class);
    when(mockJobs.get(eq(PROJECT_ID), eq(REGION_ID), eq(JOB_ID))).thenReturn(statusRequest);
    when(statusRequest.execute()).thenThrow(IOException.class);
    DataflowPipelineJob job = new DataflowPipelineJob(DataflowClient.create(options), JOB_ID, options, ImmutableMap.<AppliedPTransform<?, ?, ?>, String>of());
    long startTime = fastClock.nanoTime();
    State state = job.waitUntilFinish(Duration.standardMinutes(5), null, fastClock, fastClock);
    assertEquals(null, state);
    long timeDiff = TimeUnit.NANOSECONDS.toMillis(fastClock.nanoTime() - startTime);
    checkValidInterval(DataflowPipelineJob.MESSAGES_POLLING_INTERVAL, DataflowPipelineJob.MESSAGES_POLLING_RETRIES, timeDiff);
}