use of org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.WriteElement in project beam by apache.
the class FirestoreV1FnBatchWriteWithDeadLetterQueueTest method nonRetryableWriteIsOutput.
@Test
public void nonRetryableWriteIsOutput() throws Exception {
Write write0 = FirestoreProtoHelpers.newWrite(0);
Write write1 = FirestoreProtoHelpers.newWrite(1).toBuilder().setCurrentDocument(Precondition.newBuilder().setExists(false).build()).build();
BatchWriteRequest expectedRequest1 = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write0).addWrites(write1).build();
BatchWriteResponse response1 = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).addWriteResults(WriteResult.newBuilder().setUpdateTime(Timestamp.newBuilder().setSeconds(1).build()).build()).addStatus(statusForCode(Code.ALREADY_EXISTS)).addWriteResults(WriteResult.newBuilder().build()).build();
RpcQosOptions options = rpcQosOptions.toBuilder().withMaxAttempts(1).withBatchMaxCount(2).build();
when(processContext.element()).thenReturn(write0, write1).thenThrow(new IllegalStateException("too many calls"));
when(rpcQos.newWriteAttempt(any())).thenReturn(attempt);
when(attempt.awaitSafeToProceed(any())).thenReturn(true);
when(attempt.<Write, Element<Write>>newFlushBuffer(any())).thenReturn(newFlushBuffer(options)).thenReturn(newFlushBuffer(options)).thenThrow(new IllegalStateException("too many attempt#newFlushBuffer calls"));
when(attempt.isCodeRetryable(Code.ALREADY_EXISTS)).thenReturn(false);
ArgumentCaptor<BatchWriteRequest> requestCaptor1 = ArgumentCaptor.forClass(BatchWriteRequest.class);
when(callable.call(requestCaptor1.capture())).thenReturn(response1);
BatchWriteFnWithDeadLetterQueue fn = getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
fn.setup();
fn.startBundle(startBundleContext);
// write0
fn.processElement(processContext, window);
ArgumentCaptor<WriteFailure> writeFailureCapture = ArgumentCaptor.forClass(WriteFailure.class);
doNothing().when(processContext).outputWithTimestamp(writeFailureCapture.capture(), any());
// write1
fn.processElement(processContext, window);
WriteFailure failure = writeFailureCapture.getValue();
assertEquals(Code.ALREADY_EXISTS.getNumber(), failure.getStatus().getCode());
assertEquals(write1, failure.getWrite());
assertEquals(WriteResult.getDefaultInstance(), failure.getWriteResult());
assertEquals(expectedRequest1, requestCaptor1.getValue());
List<WriteElement> actualWrites = new ArrayList<>(fn.writes);
assertTrue(actualWrites.isEmpty());
fn.finishBundle(finishBundleContext);
verify(attempt, times(1)).recordRequestStart(any(), eq(2));
verify(attempt, times(1)).recordWriteCounts(any(), eq(1), eq(1));
verify(attempt, never()).completeSuccess();
verify(callable, times(1)).call(any());
verifyNoMoreInteractions(callable);
}
use of org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.WriteElement in project beam by apache.
the class BaseFirestoreV1WriteFnTest method endToEnd_exhaustingAttemptsResultsInException.
@Test
public final void endToEnd_exhaustingAttemptsResultsInException() throws Exception {
ApiException err1 = ApiExceptionFactory.createException(new IOException("err1"), GrpcStatusCode.of(io.grpc.Status.Code.ABORTED), false);
ApiException err2 = ApiExceptionFactory.createException(new IOException("err2"), GrpcStatusCode.of(io.grpc.Status.Code.ABORTED), false);
ApiException err3 = ApiExceptionFactory.createException(new IOException("err3"), GrpcStatusCode.of(io.grpc.Status.Code.ABORTED), false);
Instant attemptStart = Instant.ofEpochMilli(0);
Instant rpc1Start = Instant.ofEpochMilli(1);
Instant rpc1End = Instant.ofEpochMilli(2);
Instant rpc2Start = Instant.ofEpochMilli(3);
Instant rpc2End = Instant.ofEpochMilli(4);
Instant rpc3Start = Instant.ofEpochMilli(5);
Instant rpc3End = Instant.ofEpochMilli(6);
Write write = newWrite();
Element<Write> element1 = new WriteElement(0, write, window);
FlushBuffer<Element<Write>> flushBuffer = spy(newFlushBuffer(rpcQosOptions));
when(processContext.element()).thenReturn(write);
when(attempt.awaitSafeToProceed(any())).thenReturn(true);
when(attempt.<Write, Element<Write>>newFlushBuffer(attemptStart)).thenReturn(flushBuffer);
when(flushBuffer.isFull()).thenReturn(true);
when(flushBuffer.offer(element1)).thenReturn(true);
when(flushBuffer.iterator()).thenReturn(newArrayList(element1).iterator());
when(flushBuffer.getBufferedElementsCount()).thenReturn(1);
when(callable.call(any())).thenThrow(err1, err2, err3);
doNothing().when(attempt).checkCanRetry(any(), eq(err1));
doNothing().when(attempt).checkCanRetry(any(), eq(err2));
doThrow(err3).when(attempt).checkCanRetry(any(), eq(err3));
try {
FnT fn = getFn(clock, ff, rpcQosOptions, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
runFunction(fn);
fail("Expected exception");
} catch (ApiException e) {
assertNotNull(e.getMessage());
assertTrue(e.getMessage().contains("err3"));
}
verify(flushBuffer, times(1)).offer(element1);
verify(flushBuffer, atLeastOnce()).isFull();
verify(attempt, times(1)).recordRequestStart(rpc1Start, 1);
verify(attempt, times(1)).recordWriteCounts(rpc1End, 0, 1);
verify(attempt, times(1)).recordRequestStart(rpc2Start, 1);
verify(attempt, times(1)).recordWriteCounts(rpc2End, 0, 1);
verify(attempt, times(1)).recordRequestStart(rpc3Start, 1);
verify(attempt, times(1)).recordWriteCounts(rpc3End, 0, 1);
verify(attempt, never()).recordWriteCounts(any(), gt(0), anyInt());
verify(attempt, never()).completeSuccess();
}
use of org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.WriteElement in project beam by apache.
the class BaseFirestoreV1WriteFnTest method endToEnd_success.
@Test
public final void endToEnd_success() throws Exception {
Write write = newWrite();
BatchWriteRequest expectedRequest = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write).build();
BatchWriteResponse response = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).build();
Element<Write> element1 = new WriteElement(0, write, window);
Instant attemptStart = Instant.ofEpochMilli(0);
Instant rpcStart = Instant.ofEpochMilli(1);
Instant rpcEnd = Instant.ofEpochMilli(2);
RpcQosOptions options = rpcQosOptions.toBuilder().withBatchMaxCount(1).build();
FlushBuffer<Element<Write>> flushBuffer = spy(newFlushBuffer(options));
when(processContext.element()).thenReturn(write);
when(attempt.awaitSafeToProceed(any())).thenReturn(true);
when(attempt.<Write, Element<Write>>newFlushBuffer(attemptStart)).thenReturn(flushBuffer);
ArgumentCaptor<BatchWriteRequest> requestCaptor = ArgumentCaptor.forClass(BatchWriteRequest.class);
when(callable.call(requestCaptor.capture())).thenReturn(response);
runFunction(getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT));
assertEquals(expectedRequest, requestCaptor.getValue());
verify(flushBuffer, times(1)).offer(element1);
verify(flushBuffer, times(1)).isFull();
verify(attempt, times(1)).recordRequestStart(rpcStart, 1);
verify(attempt, times(1)).recordWriteCounts(rpcEnd, 1, 0);
verify(attempt, never()).recordWriteCounts(any(), anyInt(), gt(0));
verify(attempt, never()).checkCanRetry(any(), any());
}
use of org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.WriteElement in project beam by apache.
the class BaseFirestoreV1WriteFnTest method queuedWritesMaintainPriorityIfNotFlushed.
@Test
public final void queuedWritesMaintainPriorityIfNotFlushed() throws Exception {
RpcQosOptions options = rpcQosOptions.toBuilder().withMaxAttempts(1).build();
Write write0 = newWrite(0);
Write write1 = newWrite(1);
Write write2 = newWrite(2);
Write write3 = newWrite(3);
Write write4 = newWrite(4);
Instant write4Start = Instant.ofEpochMilli(4);
when(processContext.element()).thenReturn(write0, write1, write2, write3, write4).thenThrow(new IllegalStateException("too many calls"));
when(rpcQos.newWriteAttempt(any())).thenReturn(attempt);
when(attempt.awaitSafeToProceed(any())).thenReturn(true);
when(attempt.<Write, Element<Write>>newFlushBuffer(any())).thenAnswer(invocation -> newFlushBuffer(options));
FnT fn = getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
fn.setup();
fn.startBundle(startBundleContext);
// write0
fn.processElement(processContext, window);
// write1
fn.processElement(processContext, window);
// write2
fn.processElement(processContext, window);
// write3
fn.processElement(processContext, window);
// write4
fn.processElement(processContext, window);
List<WriteElement> expectedWrites = newArrayList(new WriteElement(0, write0, window), new WriteElement(1, write1, window), new WriteElement(2, write2, window), new WriteElement(3, write3, window), new WriteElement(4, write4, window));
List<WriteElement> actualWrites = new ArrayList<>(fn.writes);
assertEquals(expectedWrites, actualWrites);
assertEquals(5, fn.queueNextEntryPriority);
verify(attempt, times(1)).newFlushBuffer(write4Start);
verifyNoMoreInteractions(callable);
}
use of org.apache.beam.sdk.io.gcp.firestore.FirestoreV1WriteFn.WriteElement in project beam by apache.
the class BaseFirestoreV1WriteFnTest method attemptsExhaustedForRetryableError.
@Override
@Test
public final void attemptsExhaustedForRetryableError() throws Exception {
Instant attemptStart = Instant.ofEpochMilli(0);
Instant rpc1Start = Instant.ofEpochMilli(1);
Instant rpc1End = Instant.ofEpochMilli(2);
Instant rpc2Start = Instant.ofEpochMilli(3);
Instant rpc2End = Instant.ofEpochMilli(4);
Instant rpc3Start = Instant.ofEpochMilli(5);
Instant rpc3End = Instant.ofEpochMilli(6);
Write write = newWrite();
Element<Write> element1 = new WriteElement(0, write, window);
when(ff.getFirestoreStub(any())).thenReturn(stub);
when(ff.getRpcQos(any())).thenReturn(rpcQos);
when(rpcQos.newWriteAttempt(FirestoreV1RpcAttemptContexts.V1FnRpcAttemptContext.BatchWrite)).thenReturn(attempt);
when(stub.batchWriteCallable()).thenReturn(callable);
FlushBuffer<Element<Write>> flushBuffer = spy(newFlushBuffer(rpcQosOptions));
when(attempt.awaitSafeToProceed(any())).thenReturn(true);
when(attempt.<Write, Element<Write>>newFlushBuffer(attemptStart)).thenReturn(flushBuffer);
when(flushBuffer.offer(element1)).thenReturn(true);
when(flushBuffer.iterator()).thenReturn(newArrayList(element1).iterator());
when(flushBuffer.getBufferedElementsCount()).thenReturn(1);
when(flushBuffer.isFull()).thenReturn(true);
when(callable.call(any())).thenThrow(RETRYABLE_ERROR, RETRYABLE_ERROR, RETRYABLE_ERROR);
doNothing().when(attempt).recordWriteCounts(any(), anyInt(), anyInt());
doNothing().doNothing().doThrow(RETRYABLE_ERROR).when(attempt).checkCanRetry(any(), eq(RETRYABLE_ERROR));
when(processContext.element()).thenReturn(write);
try {
runFunction(getFn(clock, ff, rpcQosOptions, CounterFactory.DEFAULT, DistributionFactory.DEFAULT));
fail("Expected ApiException to be throw after exhausted attempts");
} catch (ApiException e) {
assertSame(RETRYABLE_ERROR, e);
}
verify(attempt, times(1)).awaitSafeToProceed(attemptStart);
verify(attempt, times(1)).recordRequestStart(rpc1Start, 1);
verify(attempt, times(1)).recordWriteCounts(rpc1End, 0, 1);
verify(attempt, times(1)).recordRequestStart(rpc2Start, 1);
verify(attempt, times(1)).recordWriteCounts(rpc2End, 0, 1);
verify(attempt, times(1)).recordRequestStart(rpc3Start, 1);
verify(attempt, times(1)).recordWriteCounts(rpc3End, 0, 1);
verify(attempt, times(0)).recordWriteCounts(any(), gt(0), anyInt());
verify(attempt, never()).completeSuccess();
}
Aggregations