Examples with Element org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element used on opensource projects

Example 6 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class RpcQosSimulationTest method safeToProceedAndWithBudgetAndWrite.

private void safeToProceedAndWithBudgetAndWrite(RpcQosImpl qos, Instant t, int expectedBatchMaxCount, int writeCount, String description) throws InterruptedException {
    RpcWriteAttemptImpl attempt = qos.newWriteAttempt(rpcAttemptContext);
    assertTrue(msg(description, t, "awaitSafeToProceed was false, expected true"), attempt.awaitSafeToProceed(t));
    FlushBufferImpl<Object, Element<Object>> buffer = attempt.newFlushBuffer(t);
    assertEquals(msg(description, t, "unexpected batchMaxCount"), expectedBatchMaxCount, buffer.nextBatchMaxCount);
    attempt.recordRequestStart(t, writeCount);
    attempt.recordWriteCounts(t, writeCount, 0);
}

Example 7 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class RpcQosTest method doTest_shouldFlush_numWritesHigherThanBatchCount.

private void doTest_shouldFlush_numWritesHigherThanBatchCount(boolean expectFlush, int batchCount, Consumer<RpcQos> preAttempt) {
    RpcQosOptions rpcQosOptions = options.toBuilder().withBatchInitialCount(10).withBatchMaxCount(10).unsafeBuild();
    RpcQos qos = new RpcQosImpl(rpcQosOptions, random, sleeper, counterFactory, distributionFactory);
    preAttempt.accept(qos);
    RpcWriteAttempt attempt = qos.newWriteAttempt(RPC_ATTEMPT_CONTEXT);
    FlushBuffer<Element<Write>> accumulator = attempt.newFlushBuffer(monotonicClock.instant());
    for (int i = 0; i < batchCount; i++) {
        accumulator.offer(new WriteElement(i, newWrite(), window));
    }
    if (expectFlush) {
        assertTrue(accumulator.isFull());
        assertEquals(10, accumulator.getBufferedElementsCount());
    } else {
        assertFalse(accumulator.isFull());
        assertEquals(batchCount, accumulator.getBufferedElementsCount());
    }
}

Example 8 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class RpcQosTest method doTest_initialBatchSizeRelativeToWorkerCount.

private void doTest_initialBatchSizeRelativeToWorkerCount(int hintWorkerCount, int expectedBatchMaxCount) {
    RpcQosOptions options = RpcQosOptions.newBuilder().withHintMaxNumWorkers(hintWorkerCount).withBatchInitialCount(500).build();
    RpcQosImpl qos = new RpcQosImpl(options, random, sleeper, counterFactory, distributionFactory);
    RpcWriteAttemptImpl attempt = qos.newWriteAttempt(RPC_ATTEMPT_CONTEXT);
    FlushBufferImpl<Object, Element<Object>> buffer = attempt.newFlushBuffer(Instant.EPOCH);
    assertEquals(expectedBatchMaxCount, buffer.nextBatchMaxCount);
}

Example 9 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class BaseFirestoreV1WriteFnTest method endToEnd_partialSuccessReturnsWritesToQueue.

@Test
public final void endToEnd_partialSuccessReturnsWritesToQueue() throws Exception {
    Write write0 = newWrite(0);
    Write write1 = newWrite(1);
    Write write2 = newWrite(2);
    Write write3 = newWrite(3);
    Write write4 = newWrite(4);
    BatchWriteRequest expectedRequest1 = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write0).addWrites(write1).addWrites(write2).addWrites(write3).addWrites(write4).build();
    BatchWriteResponse response1 = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).addStatus(statusForCode(Code.INVALID_ARGUMENT)).addStatus(statusForCode(Code.FAILED_PRECONDITION)).addStatus(statusForCode(Code.UNAUTHENTICATED)).addStatus(STATUS_OK).build();
    BatchWriteRequest expectedRequest2 = BatchWriteRequest.newBuilder().setDatabase("projects/testing-project/databases/(default)").addWrites(write1).addWrites(write2).addWrites(write3).build();
    BatchWriteResponse response2 = BatchWriteResponse.newBuilder().addStatus(STATUS_OK).addStatus(STATUS_OK).addStatus(STATUS_OK).build();
    RpcQosOptions options = rpcQosOptions.toBuilder().withMaxAttempts(1).withBatchMaxCount(5).build();
    when(processContext.element()).thenReturn(write0, write1, write2, write3, write4).thenThrow(new IllegalStateException("too many calls"));
    when(rpcQos.newWriteAttempt(any())).thenReturn(attempt);
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(any())).thenAnswer(invocation -> newFlushBuffer(options));
    when(attempt.isCodeRetryable(Code.INVALID_ARGUMENT)).thenReturn(true);
    when(attempt.isCodeRetryable(Code.FAILED_PRECONDITION)).thenReturn(true);
    when(attempt.isCodeRetryable(Code.UNAUTHENTICATED)).thenReturn(true);
    ArgumentCaptor<BatchWriteRequest> requestCaptor1 = ArgumentCaptor.forClass(BatchWriteRequest.class);
    when(callable.call(requestCaptor1.capture())).thenReturn(response1);
    FnT fn = getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
    fn.setup();
    fn.startBundle(startBundleContext);
    // write0
    fn.processElement(processContext, window);
    // write1
    fn.processElement(processContext, window);
    // write2
    fn.processElement(processContext, window);
    // write3
    fn.processElement(processContext, window);
    // write4
    fn.processElement(processContext, window);
    assertEquals(expectedRequest1, requestCaptor1.getValue());
    List<WriteElement> expectedRemainingWrites = newArrayList(new WriteElement(1, write1, window), new WriteElement(2, write2, window), new WriteElement(3, write3, window));
    List<WriteElement> actualWrites = new ArrayList<>(fn.writes);
    assertEquals(expectedRemainingWrites, actualWrites);
    assertEquals(5, fn.queueNextEntryPriority);
    ArgumentCaptor<BatchWriteRequest> requestCaptor2 = ArgumentCaptor.forClass(BatchWriteRequest.class);
    when(callable.call(requestCaptor2.capture())).thenReturn(response2);
    fn.finishBundle(finishBundleContext);
    assertEquals(expectedRequest2, requestCaptor2.getValue());
    assertEquals(0, fn.queueNextEntryPriority);
    verify(attempt, times(1)).recordRequestStart(any(), eq(5));
    verify(attempt, times(1)).recordWriteCounts(any(), eq(2), eq(3));
    verify(attempt, times(1)).recordRequestStart(any(), eq(3));
    verify(attempt, times(1)).recordWriteCounts(any(), eq(3), eq(0));
    verify(attempt, times(1)).completeSuccess();
    verify(callable, times(2)).call(any());
    verifyNoMoreInteractions(callable);
}

Example 10 with Element

use of org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element in project beam by apache.

the class BaseFirestoreV1WriteFnTest method queuedWritesMaintainPriorityIfNotFlushed.

@Test
public final void queuedWritesMaintainPriorityIfNotFlushed() throws Exception {
    RpcQosOptions options = rpcQosOptions.toBuilder().withMaxAttempts(1).build();
    Write write0 = newWrite(0);
    Write write1 = newWrite(1);
    Write write2 = newWrite(2);
    Write write3 = newWrite(3);
    Write write4 = newWrite(4);
    Instant write4Start = Instant.ofEpochMilli(4);
    when(processContext.element()).thenReturn(write0, write1, write2, write3, write4).thenThrow(new IllegalStateException("too many calls"));
    when(rpcQos.newWriteAttempt(any())).thenReturn(attempt);
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    when(attempt.<Write, Element<Write>>newFlushBuffer(any())).thenAnswer(invocation -> newFlushBuffer(options));
    FnT fn = getFn(clock, ff, options, CounterFactory.DEFAULT, DistributionFactory.DEFAULT);
    fn.setup();
    fn.startBundle(startBundleContext);
    // write0
    fn.processElement(processContext, window);
    // write1
    fn.processElement(processContext, window);
    // write2
    fn.processElement(processContext, window);
    // write3
    fn.processElement(processContext, window);
    // write4
    fn.processElement(processContext, window);
    List<WriteElement> expectedWrites = newArrayList(new WriteElement(0, write0, window), new WriteElement(1, write1, window), new WriteElement(2, write2, window), new WriteElement(3, write3, window), new WriteElement(4, write4, window));
    List<WriteElement> actualWrites = new ArrayList<>(fn.writes);
    assertEquals(expectedWrites, actualWrites);
    assertEquals(5, fn.queueNextEntryPriority);
    verify(attempt, times(1)).newFlushBuffer(write4Start);
    verifyNoMoreInteractions(callable);
}