Search in sources :

Example 1 with ListDocumentsRequest

use of com.google.firestore.v1.ListDocumentsRequest in project beam by apache.

the class FirestoreV1FnListDocumentsTest method resumeFromLastReadValue.

@Override
public void resumeFromLastReadValue() throws Exception {
    when(ff.getFirestoreStub(any())).thenReturn(stub);
    when(ff.getRpcQos(any())).thenReturn(rpcQos);
    when(rpcQos.newReadAttempt(any())).thenReturn(attempt);
    when(attempt.awaitSafeToProceed(any())).thenReturn(true);
    // First page of the response
    ListDocumentsRequest request1 = ListDocumentsRequest.newBuilder().setParent(String.format("projects/%s/databases/(default)/document", projectId)).build();
    ListDocumentsResponse response1 = ListDocumentsResponse.newBuilder().addDocuments(Document.newBuilder().setName("doc_1-1").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).addDocuments(Document.newBuilder().setName("doc_1-2").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).addDocuments(Document.newBuilder().setName("doc_1-3").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).setNextPageToken("page2").build();
    when(page1.getNextPageToken()).thenReturn(response1.getNextPageToken());
    when(page1.getResponse()).thenReturn(response1);
    when(page1.hasNextPage()).thenReturn(true);
    when(callable.call(request1)).thenReturn(pagedResponse1);
    doNothing().when(attempt).checkCanRetry(any(), eq(RETRYABLE_ERROR));
    when(pagedResponse1.iteratePages()).thenAnswer(invocation -> new Iterable<ListDocumentsPage>() {

        @Override
        public Iterator<ListDocumentsPage> iterator() {
            return new AbstractIterator<ListDocumentsPage>() {

                private boolean first = true;

                @Override
                protected ListDocumentsPage computeNext() {
                    if (first) {
                        first = false;
                        return page1;
                    } else {
                        throw RETRYABLE_ERROR;
                    }
                }
            };
        }
    });
    // Second page of the response
    ListDocumentsRequest request2 = ListDocumentsRequest.newBuilder().setParent(String.format("projects/%s/databases/(default)/document", projectId)).setPageToken("page2").build();
    ListDocumentsResponse response2 = ListDocumentsResponse.newBuilder().addDocuments(Document.newBuilder().setName("doc_2-1").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).build();
    when(page2.getResponse()).thenReturn(response2);
    when(page2.hasNextPage()).thenReturn(false);
    when(callable.call(request2)).thenReturn(pagedResponse2);
    when(pagedResponse2.iteratePages()).thenReturn(ImmutableList.of(page2));
    when(stub.listDocumentsPagedCallable()).thenReturn(callable);
    when(ff.getFirestoreStub(any())).thenReturn(stub);
    ArgumentCaptor<ListDocumentsResponse> responses = ArgumentCaptor.forClass(ListDocumentsResponse.class);
    doNothing().when(processContext).output(responses.capture());
    when(processContext.element()).thenReturn(request1);
    ListDocumentsFn fn = new ListDocumentsFn(clock, ff, rpcQosOptions);
    runFunction(fn);
    List<ListDocumentsResponse> expected = newArrayList(response1, response2);
    List<ListDocumentsResponse> allValues = responses.getAllValues();
    assertEquals(expected, allValues);
}
Also used : ListDocumentsRequest(com.google.firestore.v1.ListDocumentsRequest) ListDocumentsFn(org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.ListDocumentsFn) ListDocumentsResponse(com.google.firestore.v1.ListDocumentsResponse) ListDocumentsPage(com.google.cloud.firestore.v1.FirestoreClient.ListDocumentsPage) Iterator(java.util.Iterator) AbstractIterator(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator)

Example 2 with ListDocumentsRequest

use of com.google.firestore.v1.ListDocumentsRequest in project beam by apache.

the class FirestoreV1FnListDocumentsTest method endToEnd.

@Test
public void endToEnd() throws Exception {
    // First page of the response
    ListDocumentsRequest request1 = ListDocumentsRequest.newBuilder().setParent(String.format("projects/%s/databases/(default)/document", projectId)).build();
    ListDocumentsResponse response1 = ListDocumentsResponse.newBuilder().addDocuments(Document.newBuilder().setName("doc_1-1").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).addDocuments(Document.newBuilder().setName("doc_1-2").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).addDocuments(Document.newBuilder().setName("doc_1-3").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).setNextPageToken("page2").build();
    when(page1.getNextPageToken()).thenReturn(response1.getNextPageToken());
    when(page1.getResponse()).thenReturn(response1);
    when(page1.hasNextPage()).thenReturn(true);
    // Second page of the response
    ListDocumentsResponse response2 = ListDocumentsResponse.newBuilder().addDocuments(Document.newBuilder().setName("doc_2-1").putAllFields(ImmutableMap.of("foo", Value.newBuilder().setStringValue("bar").build())).build()).build();
    when(page2.getResponse()).thenReturn(response2);
    when(page2.hasNextPage()).thenReturn(false);
    when(pagedResponse1.iteratePages()).thenReturn(ImmutableList.of(page1, page2));
    when(callable.call(request1)).thenReturn(pagedResponse1);
    when(stub.listDocumentsPagedCallable()).thenReturn(callable);
    when(ff.getFirestoreStub(any())).thenReturn(stub);
    RpcQosOptions options = RpcQosOptions.defaultOptions();
    when(ff.getRpcQos(any())).thenReturn(FirestoreStatefulComponentFactory.INSTANCE.getRpcQos(options));
    ArgumentCaptor<ListDocumentsResponse> responses = ArgumentCaptor.forClass(ListDocumentsResponse.class);
    doNothing().when(processContext).output(responses.capture());
    when(processContext.element()).thenReturn(request1);
    ListDocumentsFn fn = new ListDocumentsFn(clock, ff, options);
    runFunction(fn);
    List<ListDocumentsResponse> expected = newArrayList(response1, response2);
    List<ListDocumentsResponse> allValues = responses.getAllValues();
    assertEquals(expected, allValues);
}
Also used : ListDocumentsRequest(com.google.firestore.v1.ListDocumentsRequest) ListDocumentsFn(org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.ListDocumentsFn) ListDocumentsResponse(com.google.firestore.v1.ListDocumentsResponse) Test(org.junit.Test)

Example 3 with ListDocumentsRequest

use of com.google.firestore.v1.ListDocumentsRequest in project beam by apache.

the class FirestoreTestingHelper method listDocumentIds.

Stream<String> listDocumentIds(String collectionPath) {
    int index = collectionPath.lastIndexOf('/');
    String parent = collectionPath.substring(0, index);
    String collectionId = collectionPath.substring(index + 1);
    ListDocumentsRequest ldr = ListDocumentsRequest.newBuilder().setParent(parent).setCollectionId(collectionId).setShowMissing(true).build();
    // LOGGER.debug("ldr = {}", ldr);
    ListDocumentsPagedResponse response = rpc.listDocumentsPagedCallable().call(ldr);
    return StreamSupport.stream(response.iteratePages().spliterator(), false).flatMap(page -> page.getResponse().getDocumentsList().stream()).map(Document::getName).filter(s -> !s.isEmpty());
}
Also used : ListDocumentsRequest(com.google.firestore.v1.ListDocumentsRequest) ListDocumentsPagedResponse(com.google.cloud.firestore.v1.FirestoreClient.ListDocumentsPagedResponse)

Aggregations

ListDocumentsRequest (com.google.firestore.v1.ListDocumentsRequest)3 ListDocumentsResponse (com.google.firestore.v1.ListDocumentsResponse)2 ListDocumentsFn (org.apache.beam.sdk.io.gcp.firestore.FirestoreV1ReadFn.ListDocumentsFn)2 ListDocumentsPage (com.google.cloud.firestore.v1.FirestoreClient.ListDocumentsPage)1 ListDocumentsPagedResponse (com.google.cloud.firestore.v1.FirestoreClient.ListDocumentsPagedResponse)1 Iterator (java.util.Iterator)1 AbstractIterator (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.AbstractIterator)1 Test (org.junit.Test)1