Examples with BackOff org.apache.beam.sdk.util.BackOff used on opensource projects

Example 1 with BackOff

use of org.apache.beam.sdk.util.BackOff in project DataflowJavaSDK-examples by GoogleCloudPlatform.

the class ExampleUtils method setup.

/**
 * Sets up external resources that are required by the example,
 * such as Pub/Sub topics and BigQuery tables.
 *
 * @throws IOException if there is a problem setting up the resources
 */
public void setup() throws IOException {
    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backOff = FluentBackoff.DEFAULT.withMaxRetries(3).withInitialBackoff(Duration.millis(200)).backoff();
    Throwable lastException = null;
    try {
        do {
            try {
                setupPubsub();
                setupBigQueryTable();
                return;
            } catch (GoogleJsonResponseException e) {
                lastException = e;
            }
        } while (BackOffUtils.next(sleeper, backOff));
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    // Ignore InterruptedException
    }
    throw new RuntimeException(lastException);
}

Example 2 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class GrpcWindmillServer method callWithBackoff.

private <ResponseT> ResponseT callWithBackoff(Supplier<ResponseT> function) {
    BackOff backoff = grpcBackoff();
    int rpcErrors = 0;
    while (true) {
        try {
            return function.get();
        } catch (StatusRuntimeException e) {
            try {
                if (++rpcErrors % 20 == 0) {
                    LOG.warn("Many exceptions calling gRPC. Last exception: {} with status {}", e, e.getStatus());
                }
                if (!BackOffUtils.next(Sleeper.DEFAULT, backoff)) {
                    throw new WindmillServerStub.RpcException(e);
                }
            } catch (IOException | InterruptedException i) {
                if (i instanceof InterruptedException) {
                    Thread.currentThread().interrupt();
                }
                WindmillServerStub.RpcException rpcException = new WindmillServerStub.RpcException(e);
                rpcException.addSuppressed(i);
                throw rpcException;
            }
        }
    }
}

Example 3 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class BaseClickHouseTest method setup.

@BeforeClass
public static void setup() throws IOException, InterruptedException {
    // network sharing doesn't work with ClassRule
    network = Network.newNetwork();
    zookeeper = new GenericContainer<>(DockerImageName.parse("zookeeper").withTag(ZOOKEEPER_VERSION)).withStartupAttempts(10).withExposedPorts(2181).withNetwork(network).withNetworkAliases("zookeeper");
    // so far zookeeper container always starts successfully, so no extra retries
    zookeeper.start();
    clickHouse = (ClickHouseContainer) new ClickHouseContainer(CLICKHOUSE_IMAGE).withStartupAttempts(10).withCreateContainerCmdModifier(// type inference for `(CreateContainerCmd) -> cmd.` doesn't work
    cmd -> ((CreateContainerCmd) cmd).withMemory(256 * 1024 * 1024L).withMemorySwap(4L * 1024 * 1024 * 1024L)).withNetwork(network).withClasspathResourceMapping("config.d/zookeeper_default.xml", "/etc/clickhouse-server/config.d/zookeeper_default.xml", BindMode.READ_ONLY);
    BackOff backOff = FluentBackoff.DEFAULT.withMaxRetries(3).withInitialBackoff(Duration.standardSeconds(15)).backoff();
    // try to start clickhouse-server a couple of times, see BEAM-6639
    while (true) {
        try {
            Unreliables.retryUntilSuccess(10, () -> {
                DockerClientFactory.instance().checkAndPullImage(DockerClientFactory.instance().client(), CLICKHOUSE_IMAGE);
                return null;
            });
            clickHouse.start();
            break;
        } catch (Exception e) {
            if (!BackOffUtils.next(Sleeper.DEFAULT, backOff)) {
                throw e;
            } else {
                List<Image> images = DockerClientFactory.instance().client().listImagesCmd().withShowAll(true).exec();
                String listImagesOutput = "listImagesCmd:\n" + Joiner.on('\n').join(images) + "\n";
                LOG.warn("failed to start clickhouse-server\n\n" + listImagesOutput, e);
            }
        }
    }
}

Example 4 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class SimplifiedKinesisClient method describeStreamSummary.

private StreamDescriptionSummary describeStreamSummary(final String streamName) throws IOException, InterruptedException {
    // DescribeStreamSummary has limits that can be hit fairly easily if we are attempting
    // to configure multiple KinesisIO inputs in the same account. Retry up to
    // DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS times if we end up hitting that limit.
    // 
    // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
    // to implement the retry policy.
    FluentBackoff retryBackoff = FluentBackoff.DEFAULT.withMaxRetries(DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS).withInitialBackoff(DESCRIBE_STREAM_SUMMARY_INITIAL_BACKOFF);
    BackOff backoff = retryBackoff.backoff();
    Sleeper sleeper = Sleeper.DEFAULT;
    DescribeStreamSummaryRequest request = new DescribeStreamSummaryRequest();
    request.setStreamName(streamName);
    while (true) {
        try {
            return kinesis.describeStreamSummary(request).getStreamDescriptionSummary();
        } catch (LimitExceededException exc) {
            if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
            }
        }
    }
}

Example 5 with BackOff

use of org.apache.beam.sdk.util.BackOff in project beam by apache.

the class RateLimitPolicyFactoryTest method defaultRateLimiterShouldUseBackoffs.

@Test
public void defaultRateLimiterShouldUseBackoffs() throws Exception {
    assertThat(withDefaultRateLimiter().getRateLimitPolicy()).isInstanceOf(DefaultRateLimiter.class);
    assertThat(withDefaultRateLimiter(millis(1), millis(1), millis(1)).getRateLimitPolicy()).isInstanceOf(DefaultRateLimiter.class);
    Sleeper sleeper = mock(Sleeper.class);
    BackOff emptySuccess = mock(BackOff.class);
    BackOff throttled = mock(BackOff.class);
    RateLimitPolicy policy = new DefaultRateLimiter(emptySuccess, throttled, sleeper);
    // reset emptySuccess after receiving at least 1 record, throttled is reset on any success
    policy.onSuccess(ImmutableList.of(mock(KinesisRecord.class)));
    verify(emptySuccess).reset();
    verify(throttled).reset();
    verifyNoInteractions(sleeper);
    clearInvocations(emptySuccess, throttled);
    when(emptySuccess.nextBackOffMillis()).thenReturn(88L, 99L);
    // throttle if no records received, throttled is reset again
    policy.onSuccess(ImmutableList.of());
    policy.onSuccess(ImmutableList.of());
    verify(emptySuccess, times(2)).nextBackOffMillis();
    verify(throttled, times(2)).reset();
    verify(sleeper).sleep(88L);
    verify(sleeper).sleep(99L);
    verifyNoMoreInteractions(sleeper, throttled, emptySuccess);
    clearInvocations(emptySuccess, throttled, sleeper);
    when(throttled.nextBackOffMillis()).thenReturn(111L, 222L);
    // throttle onThrottle
    policy.onThrottle(mock(KinesisClientThrottledException.class));
    policy.onThrottle(mock(KinesisClientThrottledException.class));
    verify(throttled, times(2)).nextBackOffMillis();
    verify(sleeper).sleep(111L);
    verify(sleeper).sleep(222L);
    verifyNoMoreInteractions(sleeper, throttled, emptySuccess);
}