Examples with BackOff org.apache.beam.sdk.util.BackOff used on opensource projects

Example 6 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class ExampleUtils method setup.

/**
 * Sets up external resources that are required by the example, such as Pub/Sub topics and
 * BigQuery tables.
 *
 * @throws IOException if there is a problem setting up the resources
 */
public void setup() throws IOException {
    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backOff = FluentBackoff.DEFAULT.withMaxRetries(3).withInitialBackoff(Duration.millis(200)).backoff();
    Throwable lastException = null;
    try {
        do {
            try {
                setupPubsub();
                setupBigQueryTable();
                return;
            } catch (GoogleJsonResponseException e) {
                lastException = e;
            }
        } while (BackOffUtils.next(sleeper, backOff));
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    // Ignore InterruptedException
    }
    throw new RuntimeException(lastException);
}

Example 7 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class RateLimitPolicyFactoryTest method defaultRateLimiterShouldUseBackoffs.

@Test
public void defaultRateLimiterShouldUseBackoffs() throws Exception {
    assertThat(withDefaultRateLimiter().getRateLimitPolicy()).isInstanceOf(DefaultRateLimiter.class);
    assertThat(withDefaultRateLimiter(millis(1), millis(1), millis(1)).getRateLimitPolicy()).isInstanceOf(DefaultRateLimiter.class);
    Sleeper sleeper = mock(Sleeper.class);
    BackOff emptySuccess = mock(BackOff.class);
    BackOff throttled = mock(BackOff.class);
    RateLimitPolicy policy = new DefaultRateLimiter(emptySuccess, throttled, sleeper);
    // reset emptySuccess after receiving at least 1 record, throttled is reset on any success
    policy.onSuccess(ImmutableList.of(mock(KinesisRecord.class)));
    verify(emptySuccess).reset();
    verify(throttled).reset();
    verifyNoInteractions(sleeper);
    clearInvocations(emptySuccess, throttled);
    when(emptySuccess.nextBackOffMillis()).thenReturn(88L, 99L);
    // throttle if no records received, throttled is reset again
    policy.onSuccess(ImmutableList.of());
    policy.onSuccess(ImmutableList.of());
    verify(emptySuccess, times(2)).nextBackOffMillis();
    verify(throttled, times(2)).reset();
    verify(sleeper).sleep(88L);
    verify(sleeper).sleep(99L);
    verifyNoMoreInteractions(sleeper, throttled, emptySuccess);
    clearInvocations(emptySuccess, throttled, sleeper);
    when(throttled.nextBackOffMillis()).thenReturn(111L, 222L);
    // throttle onThrottle
    policy.onThrottle(mock(KinesisClientThrottledException.class));
    policy.onThrottle(mock(KinesisClientThrottledException.class));
    verify(throttled, times(2)).nextBackOffMillis();
    verify(sleeper).sleep(111L);
    verify(sleeper).sleep(222L);
    verifyNoMoreInteractions(sleeper, throttled, emptySuccess);
}

Example 8 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class SimplifiedKinesisClient method describeStreamSummary.

private StreamDescriptionSummary describeStreamSummary(final String streamName) throws IOException, InterruptedException {
    // DescribeStreamSummary has limits that can be hit fairly easily if we are attempting
    // to configure multiple KinesisIO inputs in the same account. Retry up to
    // DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS times if we end up hitting that limit.
    // 
    // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
    // to implement the retry policy.
    FluentBackoff retryBackoff = FluentBackoff.DEFAULT.withMaxRetries(DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS).withInitialBackoff(DESCRIBE_STREAM_SUMMARY_INITIAL_BACKOFF);
    BackOff backoff = retryBackoff.backoff();
    Sleeper sleeper = Sleeper.DEFAULT;
    DescribeStreamSummaryRequest request = DescribeStreamSummaryRequest.builder().streamName(streamName).build();
    while (true) {
        try {
            return kinesis.describeStreamSummary(request).streamDescriptionSummary();
        } catch (LimitExceededException exc) {
            if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
            }
        }
    }
}

Example 9 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class RampupThrottlingFn method processElement.

/**
 * Emit only as many elements as the exponentially increasing budget allows.
 */
@ProcessElement
public void processElement(ProcessContext c) throws IOException, InterruptedException {
    Instant firstInstant = c.sideInput(firstInstantSideInput);
    T element = c.element();
    BackOff backoff = fluentBackoff.backoff();
    while (true) {
        Instant instant = Instant.now();
        int maxOpsBudget = calcMaxOpsBudget(firstInstant, instant, this.numWorkers.get());
        long currentOpCount = successfulOps.get(instant.getMillis());
        long availableOps = maxOpsBudget - currentOpCount;
        if (maxOpsBudget >= Integer.MAX_VALUE || availableOps > 0) {
            c.output(element);
            successfulOps.add(instant.getMillis(), 1);
            return;
        }
        long backoffMillis = backoff.nextBackOffMillis();
        LOG.info("Delaying by {}ms to conform to gradual ramp-up.", backoffMillis);
        throttlingMsecs.inc(backoffMillis);
        sleeper.sleep(backoffMillis);
    }
}