Examples with Element org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element used on opensource projects

Example 1 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class RpcQosTest method doTest_writes_shouldFlush_numBytes.

private void doTest_writes_shouldFlush_numBytes(boolean expectFlush, long numBytes) {
    RpcQosOptions rpcQosOptions = options.toBuilder().withBatchMaxBytes(3000).unsafeBuild();
    RpcQos qos = new RpcQosImpl(rpcQosOptions, random, sleeper, counterFactory, distributionFactory);
    RpcWriteAttempt attempt = qos.newWriteAttempt(RPC_ATTEMPT_CONTEXT);
    FlushBuffer<Element<Write>> accumulator = attempt.newFlushBuffer(monotonicClock.instant());
    assertTrue(accumulator.offer(new FixedSerializationSize<>(newWrite(), numBytes)));
    assertEquals(expectFlush, accumulator.isFull());
    assertEquals(numBytes, accumulator.getBufferedElementsBytes());
    assertEquals(newArrayList(newWrite()), StreamSupport.stream(accumulator.spliterator(), false).map(Element::getValue).collect(Collectors.toList()));
}

Example 2 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class RpcQosTest method offerOfElementWhichWouldCrossMaxBytesReturnFalse.

@Test
public void offerOfElementWhichWouldCrossMaxBytesReturnFalse() {
    RpcQosOptions rpcQosOptions = options.toBuilder().withBatchMaxBytes(5000).unsafeBuild();
    RpcQos qos = new RpcQosImpl(rpcQosOptions, random, sleeper, counterFactory, distributionFactory);
    RpcWriteAttempt attempt = qos.newWriteAttempt(RPC_ATTEMPT_CONTEXT);
    FlushBuffer<Element<Write>> accumulator = attempt.newFlushBuffer(monotonicClock.instant());
    assertFalse(accumulator.offer(new FixedSerializationSize<>(newWrite(), 5001)));
    assertFalse(accumulator.isFull());
    assertEquals(0, accumulator.getBufferedElementsBytes());
    assertEquals(0, accumulator.getBufferedElementsCount());
}

Example 3 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class FirestoreV1FnBatchWriteWithDeadLetterQueueTest method nonRetryableWriteIsOutput.

@Test
public void nonRetryableWriteIsOutput() throws Exception {
    Write write0 = FirestoreProtoHelpers.newWrite(0);
    Write write1 = FirestoreProtoHelpers.newWrite(1).toBuilder().setCurrentDocument(Precondition.newBuilder().setExists(false).build()).build();
    BatchWriteRequest expectedRequest1 = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write0).addWrites(write1).build();
    BatchWriteResponse response1 = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).addWriteResults(WriteResult.newBuilder().setUpdateTime(Timestamp.newBuilder().setSeconds(1).build()).build()).addStatus(statusForCode(Code.ALREADY_EXISTS)).addWriteResults(WriteResult.newBuilder().build()).build();
    RpcQosOptions options = rpcQosOptions.toBuilder().withMaxAttempts(1).withBatchMaxCount(2).build();
    when(processContext.element()).thenReturn(write0, write1).thenThrow(new IllegalStateException("too many calls"));
    when(rpcQos.newWriteAttempt(any())).thenReturn(attempt);
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(any())).thenReturn(newFlushBuffer(options)).thenReturn(newFlushBuffer(options)).thenThrow(new IllegalStateException("too many attempt#newFlushBuffer calls"));
    when(attempt.isCodeRetryable(Code.ALREADY_EXISTS)).thenReturn(false);
    ArgumentCaptor<BatchWriteRequest> requestCaptor1 = ArgumentCaptor.forClass(BatchWriteRequest.class);
    when(callable.call(requestCaptor1.capture())).thenReturn(response1);
    BatchWriteFnWithDeadLetterQueue fn = getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
    fn.setup();
    fn.startBundle(startBundleContext);
    // write0
    fn.processElement(processContext, window);
    ArgumentCaptor<WriteFailure> writeFailureCapture = ArgumentCaptor.forClass(WriteFailure.class);
    doNothing().when(processContext).outputWithTimestamp(writeFailureCapture.capture(), any());
    // write1
    fn.processElement(processContext, window);
    WriteFailure failure = writeFailureCapture.getValue();
    assertEquals(Code.ALREADY_EXISTS.getNumber(), failure.getStatus().getCode());
    assertEquals(write1, failure.getWrite());
    assertEquals(WriteResult.getDefaultInstance(), failure.getWriteResult());
    assertEquals(expectedRequest1, requestCaptor1.getValue());
    List<WriteElement> actualWrites = new ArrayList<>(fn.writes);
    assertTrue(actualWrites.isEmpty());
    fn.finishBundle(finishBundleContext);
    verify(attempt, times(1)).recordRequestStart(any(), eq(2));
    verify(attempt, times(1)).recordWriteCounts(any(), eq(1), eq(1));
    verify(attempt, never()).completeSuccess();
    verify(callable, times(1)).call(any());
    verifyNoMoreInteractions(callable);
}

Example 4 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class BaseFirestoreV1WriteFnTest method endToEnd_exhaustingAttemptsResultsInException.

@Test
public final void endToEnd_exhaustingAttemptsResultsInException() throws Exception {
    ApiException err1 = ApiExceptionFactory.createException(new IOException("err1"), GrpcStatusCode.of(io.grpc.Status.Code.ABORTED), false);
    ApiException err2 = ApiExceptionFactory.createException(new IOException("err2"), GrpcStatusCode.of(io.grpc.Status.Code.ABORTED), false);
    ApiException err3 = ApiExceptionFactory.createException(new IOException("err3"), GrpcStatusCode.of(io.grpc.Status.Code.ABORTED), false);
    Instant attemptStart = Instant.ofEpochMilli(0);
    Instant rpc1Start = Instant.ofEpochMilli(1);
    Instant rpc1End = Instant.ofEpochMilli(2);
    Instant rpc2Start = Instant.ofEpochMilli(3);
    Instant rpc2End = Instant.ofEpochMilli(4);
    Instant rpc3Start = Instant.ofEpochMilli(5);
    Instant rpc3End = Instant.ofEpochMilli(6);
    Write write = newWrite();
    Element<Write> element1 = new WriteElement(0, write, window);
    FlushBuffer<Element<Write>> flushBuffer = spy(newFlushBuffer(rpcQosOptions));
    when(processContext.element()).thenReturn(write);
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(attemptStart)).thenReturn(flushBuffer);
    when(flushBuffer.isFull()).thenReturn(true);
    when(flushBuffer.offer(element1)).thenReturn(true);
    when(flushBuffer.iterator()).thenReturn(newArrayList(element1).iterator());
    when(flushBuffer.getBufferedElementsCount()).thenReturn(1);
    when(callable.call(any())).thenThrow(err1, err2, err3);
    doNothing().when(attempt).checkCanRetry(any(), eq(err1));
    doNothing().when(attempt).checkCanRetry(any(), eq(err2));
    doThrow(err3).when(attempt).checkCanRetry(any(), eq(err3));
    try {
        FnT fn = getFn(clock, ff, rpcQosOptions, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
        runFunction(fn);
        fail("Expected exception");
    } catch (ApiException e) {
        assertNotNull(e.getMessage());
        assertTrue(e.getMessage().contains("err3"));
    }
    verify(flushBuffer, times(1)).offer(element1);
    verify(flushBuffer, atLeastOnce()).isFull();
    verify(attempt, times(1)).recordRequestStart(rpc1Start, 1);
    verify(attempt, times(1)).recordWriteCounts(rpc1End, 0, 1);
    verify(attempt, times(1)).recordRequestStart(rpc2Start, 1);
    verify(attempt, times(1)).recordWriteCounts(rpc2End, 0, 1);
    verify(attempt, times(1)).recordRequestStart(rpc3Start, 1);
    verify(attempt, times(1)).recordWriteCounts(rpc3End, 0, 1);
    verify(attempt, never()).recordWriteCounts(any(), gt(0), anyInt());
    verify(attempt, never()).completeSuccess();
}

Example 5 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class BaseFirestoreV1WriteFnTest method endToEnd_success.

@Test
public final void endToEnd_success() throws Exception {
    Write write = newWrite();
    BatchWriteRequest expectedRequest = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write).build();
    BatchWriteResponse response = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).build();
    Element<Write> element1 = new WriteElement(0, write, window);
    Instant attemptStart = Instant.ofEpochMilli(0);
    Instant rpcStart = Instant.ofEpochMilli(1);
    Instant rpcEnd = Instant.ofEpochMilli(2);
    RpcQosOptions options = rpcQosOptions.toBuilder().withBatchMaxCount(1).build();
    FlushBuffer<Element<Write>> flushBuffer = spy(newFlushBuffer(options));
    when(processContext.element()).thenReturn(write);
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(attemptStart)).thenReturn(flushBuffer);
    ArgumentCaptor<BatchWriteRequest> requestCaptor = ArgumentCaptor.forClass(BatchWriteRequest.class);
    when(callable.call(requestCaptor.capture())).thenReturn(response);
    runFunction(getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT));
    assertEquals(expectedRequest, requestCaptor.getValue());
    verify(flushBuffer, times(1)).offer(element1);
    verify(flushBuffer, times(1)).isFull();
    verify(attempt, times(1)).recordRequestStart(rpcStart, 1);
    verify(attempt, times(1)).recordWriteCounts(rpcEnd, 1, 0);
    verify(attempt, never()).recordWriteCounts(any(), anyInt(), gt(0));
    verify(attempt, never()).checkCanRetry(any(), any());
}