Examples with FluentBackoff org.apache.beam.sdk.util.FluentBackoff used on opensource projects

Example 1 with FluentBackoff

use of org.apache.beam.sdk.util.FluentBackoff in project beam by apache.

the class SimplifiedKinesisClient method describeStreamSummary.

private StreamDescriptionSummary describeStreamSummary(final String streamName) throws IOException, InterruptedException {
    // DescribeStreamSummary has limits that can be hit fairly easily if we are attempting
    // to configure multiple KinesisIO inputs in the same account. Retry up to
    // DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS times if we end up hitting that limit.
    // 
    // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
    // to implement the retry policy.
    FluentBackoff retryBackoff = FluentBackoff.DEFAULT.withMaxRetries(DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS).withInitialBackoff(DESCRIBE_STREAM_SUMMARY_INITIAL_BACKOFF);
    BackOff backoff = retryBackoff.backoff();
    Sleeper sleeper = Sleeper.DEFAULT;
    DescribeStreamSummaryRequest request = new DescribeStreamSummaryRequest();
    request.setStreamName(streamName);
    while (true) {
        try {
            return kinesis.describeStreamSummary(request).getStreamDescriptionSummary();
        } catch (LimitExceededException exc) {
            if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
            }
        }
    }
}

Example 2 with FluentBackoff

use of org.apache.beam.sdk.util.FluentBackoff in project beam by apache.

the class TrafficMaxLaneFlowIT method testE2ETrafficMaxLaneFlow.

@Test
public void testE2ETrafficMaxLaneFlow() throws Exception {
    this.options.setBigQuerySchema(FormatMaxesFn.getSchema());
    this.options.setProject(this.projectId);
    this.options.setBigQueryDataset(this.outputDatasetId);
    this.options.setBigQueryTable(this.outputTable);
    TrafficMaxLaneFlow.runTrafficMaxLaneFlow(this.options);
    FluentBackoff backoffFactory = FluentBackoff.DEFAULT.withMaxRetries(4).withInitialBackoff(Duration.standardSeconds(1L));
    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backoff = BackOffAdapter.toGcpBackOff(backoffFactory.backoff());
    String res = "empty_result";
    // Partial results are still returned making traversal of nested result object NPE prone.
    do {
        QueryResponse response = this.bqClient.queryWithRetries(String.format("SELECT count(*) as total FROM [%s:%s.%s]", this.projectId, this.outputDatasetId, this.outputTable), this.projectId);
        // Partial results are still returned making traversal of nested result object NPE prone.
        try {
            res = response.getRows().get(0).getF().get(0).getV().toString();
            break;
        } catch (NullPointerException e) {
        // Ignore NullPointerException during retry.
        }
    } while (BackOffUtils.next(sleeper, backoff));
    assertEquals("9763", res);
}

Example 3 with FluentBackoff

use of org.apache.beam.sdk.util.FluentBackoff in project beam by apache.

the class TrafficRoutesIT method testE2ETrafficRoutes.

@Test
public void testE2ETrafficRoutes() throws Exception {
    this.options.setBigQuerySchema(FormatStatsFn.getSchema());
    this.options.setProject(this.projectId);
    this.options.setBigQueryDataset(this.outputDatasetId);
    this.options.setBigQueryTable(this.outputTable);
    TrafficRoutes.runTrafficRoutes(options);
    FluentBackoff backoffFactory = FluentBackoff.DEFAULT.withMaxRetries(4).withInitialBackoff(Duration.standardSeconds(1L));
    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backoff = BackOffAdapter.toGcpBackOff(backoffFactory.backoff());
    String res = "empty_result";
    do {
        QueryResponse response = this.bqClient.queryWithRetries(String.format("SELECT count(*) as total FROM [%s:%s.%s]", this.projectId, this.outputDatasetId, this.outputTable), this.projectId);
        // Partial results are still returned making traversal of nested result object NPE prone.
        try {
            res = response.getRows().get(0).getF().get(0).getV().toString();
            break;
        } catch (NullPointerException e) {
        // Ignore NullPointerException during retry.
        }
    } while (BackOffUtils.next(sleeper, backoff));
    assertEquals("27", res);
}

Example 4 with FluentBackoff

use of org.apache.beam.sdk.util.FluentBackoff in project beam by apache.

the class SimplifiedKinesisClient method describeStreamSummary.

private StreamDescriptionSummary describeStreamSummary(final String streamName) throws IOException, InterruptedException {
    // DescribeStreamSummary has limits that can be hit fairly easily if we are attempting
    // to configure multiple KinesisIO inputs in the same account. Retry up to
    // DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS times if we end up hitting that limit.
    // 
    // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
    // to implement the retry policy.
    FluentBackoff retryBackoff = FluentBackoff.DEFAULT.withMaxRetries(DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS).withInitialBackoff(DESCRIBE_STREAM_SUMMARY_INITIAL_BACKOFF);
    BackOff backoff = retryBackoff.backoff();
    Sleeper sleeper = Sleeper.DEFAULT;
    DescribeStreamSummaryRequest request = DescribeStreamSummaryRequest.builder().streamName(streamName).build();
    while (true) {
        try {
            return kinesis.describeStreamSummary(request).streamDescriptionSummary();
        } catch (LimitExceededException exc) {
            if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
            }
        }
    }
}

Example 5 with FluentBackoff

use of org.apache.beam.sdk.util.FluentBackoff in project beam by apache.

the class BigQueryToTableIT method getTableRowsFromQuery.

private List<TableRow> getTableRowsFromQuery(String query, int maxRetry) throws Exception {
    FluentBackoff backoffFactory = FluentBackoff.DEFAULT.withMaxRetries(maxRetry).withInitialBackoff(Duration.standardSeconds(1L));
    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backoff = BackOffAdapter.toGcpBackOff(backoffFactory.backoff());
    do {
        LOG.info("Starting querying {}", query);
        QueryResponse response = BQ_CLIENT.queryWithRetries(query, project);
        if (response.getRows() != null) {
            LOG.info("Got table content with query {}", query);
            return response.getRows();
        }
    } while (BackOffUtils.next(sleeper, backoff));
    LOG.info("Got empty table for query {} with retry {}", query, maxRetry);
    return Collections.emptyList();
}