use of com.google.api.client.testing.util.MockSleeper in project beam by apache.
the class BigQueryServicesImplTest method testInsertTimeoutLog.
/**
* Tests that {@link DatasetServiceImpl#insertAll} logs suggested remedy for insert timeouts.
*/
@Test
public void testInsertTimeoutLog() throws Exception {
TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = new ArrayList<>();
rows.add(wrapValue(new TableRow()));
setupMockResponses(response -> {
when(response.getStatusCode()).thenReturn(400);
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
when(response.getContent()).thenReturn(toStream(errorWithReasonAndStatus(" No rows present in the request. ", 400)));
});
DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
RuntimeException e = assertThrows(RuntimeException.class, () -> dataService.insertAll(ref, rows, null, BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), TEST_BACKOFF, new MockSleeper(), InsertRetryPolicy.alwaysRetry(), null, null, false, false, false, null));
assertThat(e.getCause().getMessage(), containsString("No rows present in the request."));
verifyAllResponsesAreRead();
expectedLogs.verifyError("No rows present in the request error likely caused by");
verifyWriteMetricWasSet("project", "dataset", "table", " no rows present in the request. ", 1);
}
use of com.google.api.client.testing.util.MockSleeper in project beam by apache.
the class BigQueryServicesImplTest method testInsertFailsGracefully.
/**
* Tests that {@link DatasetServiceImpl#insertAll} fails gracefully when persistent issues.
*/
@Test
public void testInsertFailsGracefully() throws Exception {
TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = ImmutableList.of(wrapValue(new TableRow()), wrapValue(new TableRow()));
ErrorProto errorProto = new ErrorProto().setReason("schemaMismatch");
final TableDataInsertAllResponse row1Failed = new TableDataInsertAllResponse().setInsertErrors(ImmutableList.of(new InsertErrors().setIndex(1L).setErrors(ImmutableList.of(errorProto))));
final TableDataInsertAllResponse row0Failed = new TableDataInsertAllResponse().setInsertErrors(ImmutableList.of(new InsertErrors().setIndex(0L).setErrors(ImmutableList.of(errorProto))));
MockSetupFunction row0FailureResponseFunction = response -> {
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
// Always return 200.
when(response.getStatusCode()).thenReturn(200);
when(response.getContent()).thenAnswer(invocation -> toStream(row0Failed));
};
setupMockResponses(response -> {
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
// Always return 200.
when(response.getStatusCode()).thenReturn(200);
// Return row 1 failing, then we retry row 1 as row 0, and row 0 persistently fails.
when(response.getContent()).thenReturn(toStream(row1Failed));
}, // 3 failures
row0FailureResponseFunction, row0FailureResponseFunction, row0FailureResponseFunction);
DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
// Expect it to fail.
try {
dataService.insertAll(ref, rows, null, BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), TEST_BACKOFF, new MockSleeper(), InsertRetryPolicy.alwaysRetry(), null, null, false, false, false, null);
fail();
} catch (IOException e) {
assertThat(e, instanceOf(IOException.class));
assertThat(e.getMessage(), containsString("Insert failed:"));
assertThat(e.getMessage(), containsString("[{\"errors\":[{\"reason\":\"schemaMismatch\"}]"));
}
// Verify the exact number of retries as well as log messages.
verifyAllResponsesAreRead();
expectedLogs.verifyInfo("Retrying 1 failed inserts to BigQuery");
verifyWriteMetricWasSet("project", "dataset", "table", "schemamismatch", 4);
}
use of com.google.api.client.testing.util.MockSleeper in project beam by apache.
the class BigQueryServicesImplTest method testInsertRetryTransientsAboveRequestByteSizeLimits.
@Test
public void testInsertRetryTransientsAboveRequestByteSizeLimits() throws Exception {
TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = ImmutableList.of(wrapValue(new TableRow().set("row", Strings.repeat("abcdefghi", 1024 * 1025))), wrapValue(new TableRow().set("row", "a")), wrapValue(new TableRow().set("row", "b")));
List<String> insertIds = ImmutableList.of("a", "b", "c");
final TableDataInsertAllResponse allRowsSucceeded = new TableDataInsertAllResponse();
setupMockResponses(response -> {
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
when(response.getStatusCode()).thenReturn(200);
when(response.getContent()).thenReturn(toStream(allRowsSucceeded));
});
DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.fromArgs("--maxStreamingBatchSize=15").create());
List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
List<ValueInSingleWindow<TableRow>> successfulRows = Lists.newArrayList();
dataService.<TableRow>insertAll(ref, rows, insertIds, BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), TEST_BACKOFF, new MockSleeper(), InsertRetryPolicy.retryTransientErrors(), failedInserts, ErrorContainer.TABLE_ROW_ERROR_CONTAINER, false, false, false, successfulRows);
assertEquals(1, failedInserts.size());
assertEquals(2, successfulRows.size());
verifyAllResponsesAreRead();
verifyWriteMetricWasSet("project", "dataset", "table", "ok", 1);
}
use of com.google.api.client.testing.util.MockSleeper in project beam by apache.
the class BigQueryServicesImplTest method testInsertRateLimitRetry.
/**
* Tests that {@link DatasetServiceImpl#insertAll} retries rate limited attempts.
*/
@Test
public void testInsertRateLimitRetry() throws Exception {
TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = new ArrayList<>();
rows.add(wrapValue(new TableRow()));
// First response is 403 rate limited, second response has valid payload.
setupMockResponses(response -> {
when(response.getStatusCode()).thenReturn(403);
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
when(response.getContent()).thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)));
}, response -> {
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
when(response.getStatusCode()).thenReturn(200);
when(response.getContent()).thenReturn(toStream(new TableDataInsertAllResponse()));
});
DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
dataService.insertAll(ref, rows, null, BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), TEST_BACKOFF, new MockSleeper(), InsertRetryPolicy.alwaysRetry(), null, null, false, false, false, null);
verifyAllResponsesAreRead();
expectedLogs.verifyInfo("BigQuery insertAll error, retrying:");
verifyWriteMetricWasSet("project", "dataset", "table", "ratelimitexceeded", 1);
}
use of com.google.api.client.testing.util.MockSleeper in project beam by apache.
the class BigQueryServicesImplTest method testInsertWithinRequestByteSizeLimits.
/**
* Tests that {@link DatasetServiceImpl#insertAll} does not go over limit of rows per request.
*/
@Test
public void testInsertWithinRequestByteSizeLimits() throws Exception {
TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = ImmutableList.of(wrapValue(new TableRow().set("row", "a")), wrapValue(new TableRow().set("row", "b")), wrapValue(new TableRow().set("row", "cdefghijklmnopqrstuvwxyz")));
List<String> insertIds = ImmutableList.of("a", "b", "c");
final TableDataInsertAllResponse allRowsSucceeded = new TableDataInsertAllResponse();
setupMockResponses(response -> {
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
when(response.getStatusCode()).thenReturn(200);
when(response.getContent()).thenReturn(toStream(allRowsSucceeded));
}, response -> {
when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
when(response.getStatusCode()).thenReturn(200);
when(response.getContent()).thenReturn(toStream(allRowsSucceeded));
});
DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.fromArgs("--maxStreamingBatchSize=15").create());
dataService.insertAll(ref, rows, insertIds, BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), TEST_BACKOFF, new MockSleeper(), InsertRetryPolicy.retryTransientErrors(), new ArrayList<>(), ErrorContainer.TABLE_ROW_ERROR_CONTAINER, false, false, false, null);
verifyAllResponsesAreRead();
verifyWriteMetricWasSet("project", "dataset", "table", "ok", 2);
}
Aggregations