Examples with Write com.google.firestore.v1.Write used on opensource projects

Example 16 with Write

use of com.google.firestore.v1.Write in project beam by apache.

the class BaseFirestoreV1WriteFnTest method endToEnd_maxBatchSizeRespected.

@Test
public final void endToEnd_maxBatchSizeRespected() throws Exception {
    Instant enqueue0 = Instant.ofEpochMilli(0);
    Instant enqueue1 = Instant.ofEpochMilli(1);
    Instant enqueue2 = Instant.ofEpochMilli(2);
    Instant enqueue3 = Instant.ofEpochMilli(3);
    Instant enqueue4 = Instant.ofEpochMilli(4);
    Instant group1Rpc1Start = Instant.ofEpochMilli(5);
    Instant group1Rpc1End = Instant.ofEpochMilli(6);
    Instant enqueue5 = Instant.ofEpochMilli(7);
    Instant finalFlush = Instant.ofEpochMilli(8);
    Instant group2Rpc1Start = Instant.ofEpochMilli(9);
    Instant group2Rpc1End = Instant.ofEpochMilli(10);
    Write write0 = newWrite(0);
    Write write1 = newWrite(1);
    Write write2 = newWrite(2);
    Write write3 = newWrite(3);
    Write write4 = newWrite(4);
    Write write5 = newWrite(5);
    int maxValuesPerGroup = 5;
    BatchWriteRequest.Builder builder = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)");
    BatchWriteRequest expectedGroup1Request = builder.build().toBuilder().addWrites(write0).addWrites(write1).addWrites(write2).addWrites(write3).addWrites(write4).build();
    BatchWriteRequest expectedGroup2Request = builder.build().toBuilder().addWrites(write5).build();
    BatchWriteResponse group1Response = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).addStatus(STATUS_OK).addStatus(STATUS_OK).addStatus(STATUS_OK).addStatus(STATUS_OK).build();
    BatchWriteResponse group2Response = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).build();
    RpcQosOptions options = rpcQosOptions.toBuilder().withBatchMaxCount(maxValuesPerGroup).build();
    FlushBuffer<Element<Write>> flushBuffer = spy(newFlushBuffer(options));
    FlushBuffer<Element<Write>> flushBuffer2 = spy(newFlushBuffer(options));
    when(processContext.element()).thenReturn(write0, write1, write2, write3, write4, write5);
    when(rpcQos.newWriteAttempt(any())).thenReturn(attempt, attempt, attempt, attempt, attempt, attempt2, attempt2, attempt2).thenThrow(new IllegalStateException("too many attempts"));
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt2.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(enqueue0)).thenReturn(newFlushBuffer(options));
    when(attempt.<Write, Element<Write>>newFlushBuffer(enqueue1)).thenReturn(newFlushBuffer(options));
    when(attempt.<Write, Element<Write>>newFlushBuffer(enqueue2)).thenReturn(newFlushBuffer(options));
    when(attempt.<Write, Element<Write>>newFlushBuffer(enqueue3)).thenReturn(newFlushBuffer(options));
    when(attempt.<Write, Element<Write>>newFlushBuffer(enqueue4)).thenReturn(flushBuffer);
    when(callable.call(expectedGroup1Request)).thenReturn(group1Response);
    when(attempt2.<Write, Element<Write>>newFlushBuffer(enqueue5)).thenReturn(newFlushBuffer(options));
    when(attempt2.<Write, Element<Write>>newFlushBuffer(finalFlush)).thenReturn(flushBuffer2);
    when(callable.call(expectedGroup2Request)).thenReturn(group2Response);
    runFunction(getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT), maxValuesPerGroup + 1);
    verify(attempt, times(1)).recordRequestStart(group1Rpc1Start, 5);
    verify(attempt, times(1)).recordWriteCounts(group1Rpc1End, 5, 0);
    verify(attempt, times(1)).completeSuccess();
    verify(attempt2, times(1)).recordRequestStart(group2Rpc1Start, 1);
    verify(attempt2, times(1)).recordWriteCounts(group2Rpc1End, 1, 0);
    verify(attempt2, times(1)).completeSuccess();
    verify(callable, times(1)).call(expectedGroup1Request);
    verify(callable, times(1)).call(expectedGroup2Request);
    verifyNoMoreInteractions(callable);
    verify(flushBuffer, times(maxValuesPerGroup)).offer(any());
    verify(flushBuffer2, times(1)).offer(any());
}

Example 17 with Write

use of com.google.firestore.v1.Write in project beam by apache.

the class BaseFirestoreV1WriteFnTest method writesRemainInQueueWhenFlushIsNotReadyAndThenFlushesInFinishBundle.

@Test
public final void writesRemainInQueueWhenFlushIsNotReadyAndThenFlushesInFinishBundle() throws Exception {
    RpcQosOptions options = rpcQosOptions.toBuilder().withMaxAttempts(1).build();
    Write write = newWrite();
    BatchWriteRequest expectedRequest = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write).build();
    BatchWriteResponse response = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).build();
    when(processContext.element()).thenReturn(write).thenThrow(new IllegalStateException("too many element calls"));
    when(rpcQos.newWriteAttempt(any())).thenReturn(attempt, attempt2).thenThrow(new IllegalStateException("too many attempt calls"));
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt2.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(any())).thenAnswer(invocation -> newFlushBuffer(options));
    when(attempt2.<Write, Element<Write>>newFlushBuffer(any())).thenAnswer(invocation -> newFlushBuffer(options));
    FnT fn = getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
    fn.populateDisplayData(displayDataBuilder);
    fn.setup();
    fn.startBundle(startBundleContext);
    fn.processElement(processContext, window);
    assertEquals(1, fn.writes.size());
    verify(attempt, never()).recordWriteCounts(any(), anyInt(), anyInt());
    verify(attempt, never()).checkCanRetry(any(), any());
    verify(attempt, never()).completeSuccess();
    Instant attempt2RpcStart = Instant.ofEpochMilli(2);
    Instant attempt2RpcEnd = Instant.ofEpochMilli(3);
    ArgumentCaptor<BatchWriteRequest> requestCaptor = ArgumentCaptor.forClass(BatchWriteRequest.class);
    when(callable.call(requestCaptor.capture())).thenReturn(response);
    fn.finishBundle(finishBundleContext);
    assertEquals(0, fn.writes.size());
    assertEquals(expectedRequest, requestCaptor.getValue());
    verify(attempt2, times(1)).recordRequestStart(attempt2RpcStart, 1);
    verify(attempt2, times(1)).recordWriteCounts(attempt2RpcEnd, 1, 0);
    verify(attempt2, never()).recordWriteCounts(any(), anyInt(), gt(0));
    verify(attempt2, never()).checkCanRetry(any(), any());
    verify(attempt2, times(1)).completeSuccess();
}

Example 18 with Write

use of com.google.firestore.v1.Write in project beam by apache.

the class BaseFirestoreV1WriteFnTest method attemptsExhaustedForRetryableError.

@Override
@Test
public final void attemptsExhaustedForRetryableError() throws Exception {
    Instant attemptStart = Instant.ofEpochMilli(0);
    Instant rpc1Start = Instant.ofEpochMilli(1);
    Instant rpc1End = Instant.ofEpochMilli(2);
    Instant rpc2Start = Instant.ofEpochMilli(3);
    Instant rpc2End = Instant.ofEpochMilli(4);
    Instant rpc3Start = Instant.ofEpochMilli(5);
    Instant rpc3End = Instant.ofEpochMilli(6);
    Write write = newWrite();
    Element<Write> element1 = new WriteElement(0, write, window);
    when(ff.getFirestoreStub(any())).thenReturn(stub);
    when(ff.getRpcQos(any())).thenReturn(rpcQos);
    when(rpcQos.newWriteAttempt(FirestoreV1RpcAttemptContexts.V1FnRpcAttemptContext.BatchWrite)).thenReturn(attempt);
    when(stub.batchWriteCallable()).thenReturn(callable);
    FlushBuffer<Element<Write>> flushBuffer = spy(newFlushBuffer(rpcQosOptions));
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(attemptStart)).thenReturn(flushBuffer);
    when(flushBuffer.offer(element1)).thenReturn(true);
    when(flushBuffer.iterator()).thenReturn(newArrayList(element1).iterator());
    when(flushBuffer.getBufferedElementsCount()).thenReturn(1);
    when(flushBuffer.isFull()).thenReturn(true);
    when(callable.call(any())).thenThrow(RETRYABLE_ERROR, RETRYABLE_ERROR, RETRYABLE_ERROR);
    doNothing().when(attempt).recordWriteCounts(any(), anyInt(), anyInt());
    doNothing().doNothing().doThrow(RETRYABLE_ERROR).when(attempt).checkCanRetry(any(), eq(RETRYABLE_ERROR));
    when(processContext.element()).thenReturn(write);
    try {
        runFunction(getFn(clock, ff, rpcQosOptions, CounterFactory.DEFAULT, DistributionFactory.DEFAULT));
        fail("Expected ApiException to be throw after exhausted attempts");
    } catch (ApiException e) {
        assertSame(RETRYABLE_ERROR, e);
    }
    verify(attempt, times(1)).awaitSafeToProceed(attemptStart);
    verify(attempt, times(1)).recordRequestStart(rpc1Start, 1);
    verify(attempt, times(1)).recordWriteCounts(rpc1End, 0, 1);
    verify(attempt, times(1)).recordRequestStart(rpc2Start, 1);
    verify(attempt, times(1)).recordWriteCounts(rpc2End, 0, 1);
    verify(attempt, times(1)).recordRequestStart(rpc3Start, 1);
    verify(attempt, times(1)).recordWriteCounts(rpc3End, 0, 1);
    verify(attempt, times(0)).recordWriteCounts(any(), gt(0), anyInt());
    verify(attempt, never()).completeSuccess();
}

Example 19 with Write

use of com.google.firestore.v1.Write in project beam by apache.

the class BaseFirestoreV1WriteFnTest method endToEnd_partialSuccessReturnsWritesToQueue.

@Test
public final void endToEnd_partialSuccessReturnsWritesToQueue() throws Exception {
    Write write0 = newWrite(0);
    Write write1 = newWrite(1);
    Write write2 = newWrite(2);
    Write write3 = newWrite(3);
    Write write4 = newWrite(4);
    BatchWriteRequest expectedRequest1 = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write0).addWrites(write1).addWrites(write2).addWrites(write3).addWrites(write4).build();
    BatchWriteResponse response1 = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).addStatus(statusForCode(Code.INVALID_ARGUMENT)).addStatus(statusForCode(Code.FAILED_PRECONDITION)).addStatus(statusForCode(Code.UNAUTHENTICATED)).addStatus(STATUS_OK).build();
    BatchWriteRequest expectedRequest2 = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write1).addWrites(write2).addWrites(write3).build();
    BatchWriteResponse response2 = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).addStatus(STATUS_OK).addStatus(STATUS_OK).build();
    RpcQosOptions options = rpcQosOptions.toBuilder().withMaxAttempts(1).withBatchMaxCount(5).build();
    when(processContext.element()).thenReturn(write0, write1, write2, write3, write4).thenThrow(new IllegalStateException("too many calls"));
    when(rpcQos.newWriteAttempt(any())).thenReturn(attempt);
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(any())).thenAnswer(invocation -> newFlushBuffer(options));
    when(attempt.isCodeRetryable(Code.INVALID_ARGUMENT)).thenReturn(true);
    when(attempt.isCodeRetryable(Code.FAILED_PRECONDITION)).thenReturn(true);
    when(attempt.isCodeRetryable(Code.UNAUTHENTICATED)).thenReturn(true);
    ArgumentCaptor<BatchWriteRequest> requestCaptor1 = ArgumentCaptor.forClass(BatchWriteRequest.class);
    when(callable.call(requestCaptor1.capture())).thenReturn(response1);
    FnT fn = getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
    fn.setup();
    fn.startBundle(startBundleContext);
    // write0
    fn.processElement(processContext, window);
    // write1
    fn.processElement(processContext, window);
    // write2
    fn.processElement(processContext, window);
    // write3
    fn.processElement(processContext, window);
    // write4
    fn.processElement(processContext, window);
    assertEquals(expectedRequest1, requestCaptor1.getValue());
    List<WriteElement> expectedRemainingWrites = newArrayList(new WriteElement(1, write1, window), new WriteElement(2, write2, window), new WriteElement(3, write3, window));
    List<WriteElement> actualWrites = new ArrayList<>(fn.writes);
    assertEquals(expectedRemainingWrites, actualWrites);
    assertEquals(5, fn.queueNextEntryPriority);
    ArgumentCaptor<BatchWriteRequest> requestCaptor2 = ArgumentCaptor.forClass(BatchWriteRequest.class);
    when(callable.call(requestCaptor2.capture())).thenReturn(response2);
    fn.finishBundle(finishBundleContext);
    assertEquals(expectedRequest2, requestCaptor2.getValue());
    assertEquals(0, fn.queueNextEntryPriority);
    verify(attempt, times(1)).recordRequestStart(any(), eq(5));
    verify(attempt, times(1)).recordWriteCounts(any(), eq(2), eq(3));
    verify(attempt, times(1)).recordRequestStart(any(), eq(3));
    verify(attempt, times(1)).recordWriteCounts(any(), eq(3), eq(0));
    verify(attempt, times(1)).completeSuccess();
    verify(callable, times(2)).call(any());
    verifyNoMoreInteractions(callable);
}

Example 20 with Write

use of com.google.firestore.v1.Write in project beam by apache.

the class FirestoreV1FnBatchWriteWithDeadLetterQueueTest method enqueueingWritesValidateBytesSize.

@Override
@Test
public void enqueueingWritesValidateBytesSize() throws Exception {
    int maxBytes = 50;
    RpcQosOptions options = rpcQosOptions.toBuilder().withBatchMaxBytes(maxBytes).build();
    when(ff.getFirestoreStub(any())).thenReturn(stub);
    when(ff.getRpcQos(any())).thenReturn(FirestoreStatefulComponentFactory.INSTANCE.getRpcQos(options));
    byte[] bytes = new byte[maxBytes + 1];
    SecureRandom.getInstanceStrong().nextBytes(bytes);
    byte[] base64Bytes = Base64.getEncoder().encode(bytes);
    String base64String = Base64.getEncoder().encodeToString(bytes);
    Value largeValue = Value.newBuilder().setStringValueBytes(ByteString.copyFrom(base64Bytes)).build();
    // apply a doc transform that is too large
    Write write1 = Write.newBuilder().setTransform(DocumentTransform.newBuilder().setDocument(String.format("doc-%03d", 2)).addFieldTransforms(FieldTransform.newBuilder().setAppendMissingElements(ArrayValue.newBuilder().addValues(largeValue)))).build();
    // delete a doc that is too large
    Write write2 = Write.newBuilder().setDelete(String.format("doc-%03d_%s", 3, base64String)).build();
    // update a doc that is too large
    Write write3 = Write.newBuilder().setUpdate(Document.newBuilder().setName(String.format("doc-%03d", 4)).putAllFields(ImmutableMap.of("foo", largeValue))).build();
    BatchWriteFnWithDeadLetterQueue fn = getFn(clock, ff, options, metricsFixture.counterFactory, metricsFixture.distributionFactory);
    fn.populateDisplayData(displayDataBuilder);
    fn.setup();
    fn.startBundle(startBundleContext);
    ArgumentCaptor<WriteFailure> write1FailureCapture = ArgumentCaptor.forClass(WriteFailure.class);
    doNothing().when(processContext).outputWithTimestamp(write1FailureCapture.capture(), any());
    when(processContext.element()).thenReturn(write1);
    fn.processElement(processContext, window);
    WriteFailure failure = write1FailureCapture.getValue();
    assertNotNull(failure);
    String message = failure.getStatus().getMessage();
    assertTrue(message.contains("TRANSFORM"));
    assertTrue(message.contains("larger than configured max allowed bytes per batch"));
    ArgumentCaptor<WriteFailure> write2FailureCapture = ArgumentCaptor.forClass(WriteFailure.class);
    doNothing().when(processContext).outputWithTimestamp(write2FailureCapture.capture(), any());
    when(processContext.element()).thenReturn(write2);
    fn.processElement(processContext, window);
    WriteFailure failure2 = write2FailureCapture.getValue();
    assertNotNull(failure2);
    String message2 = failure2.getStatus().getMessage();
    assertTrue(message2.contains("DELETE"));
    assertTrue(message2.contains("larger than configured max allowed bytes per batch"));
    ArgumentCaptor<WriteFailure> write3FailureCapture = ArgumentCaptor.forClass(WriteFailure.class);
    doNothing().when(processContext).outputWithTimestamp(write3FailureCapture.capture(), any());
    when(processContext.element()).thenReturn(write3);
    fn.processElement(processContext, window);
    WriteFailure failure3 = write3FailureCapture.getValue();
    assertNotNull(failure3);
    String message3 = failure3.getStatus().getMessage();
    assertTrue(message3.contains("UPDATE"));
    assertTrue(message3.contains("larger than configured max allowed bytes per batch"));
    assertEquals(0, fn.writes.size());
}