Search in sources :

Example 1 with RpcQosOptions

use of org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions in project beam by apache.

the class BaseFirestoreIT method listCollections.

@Test
@TestDataLayoutHint(DataLayout.Deep)
public final void listCollections() throws Exception {
    // verification and cleanup of nested collections is much slower because each document
    // requires an rpc to find its collections, instead of using the usual size, use 20
    // to keep the test quick
    List<String> collectionIds = IntStream.rangeClosed(1, 20).mapToObj(i -> helper.colId()).collect(Collectors.toList());
    ApiFutures.transform(ApiFutures.allAsList(chunkUpDocIds(collectionIds).map(chunk -> {
        WriteBatch batch = helper.getFs().batch();
        chunk.stream().map(col -> helper.getBaseDocument().collection(col).document()).forEach(ref -> batch.set(ref, ImmutableMap.of("foo", "bar")));
        return batch.commit();
    }).collect(Collectors.toList())), FirestoreTestingHelper.flattenListList(), MoreExecutors.directExecutor()).get(10, TimeUnit.SECONDS);
    PCollection<String> actualCollectionIds = testPipeline.apply(Create.of("")).apply(getListCollectionIdsPTransform(testName.getMethodName())).apply(FirestoreIO.v1().read().listCollectionIds().withRpcQosOptions(rpcQosOptions).build());
    PAssert.that(actualCollectionIds).containsInAnyOrder(collectionIds);
    testPipeline.run(options);
}
Also used : IntStream(java.util.stream.IntStream) BatchGetDocumentsResponse(com.google.firestore.v1.BatchGetDocumentsResponse) Assume.assumeThat(org.junit.Assume.assumeThat) RunQueryRequest(com.google.firestore.v1.RunQueryRequest) Write(com.google.firestore.v1.Write) BeforeClass(org.junit.BeforeClass) TestDataLayoutHint(org.apache.beam.sdk.io.gcp.firestore.it.FirestoreTestingHelper.TestDataLayoutHint) CleanupMode(org.apache.beam.sdk.io.gcp.firestore.it.FirestoreTestingHelper.CleanupMode) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) BatchGetDocumentsRequest(com.google.firestore.v1.BatchGetDocumentsRequest) PTransform(org.apache.beam.sdk.transforms.PTransform) DocumentGenerator(org.apache.beam.sdk.io.gcp.firestore.it.FirestoreTestingHelper.DocumentGenerator) Create(org.apache.beam.sdk.transforms.Create) TestName(org.junit.rules.TestName) TestPipeline(org.apache.beam.sdk.testing.TestPipeline) FirestoreOptions(org.apache.beam.sdk.io.gcp.firestore.FirestoreOptions) Timeout(org.junit.rules.Timeout) ListCollectionIdsRequest(com.google.firestore.v1.ListCollectionIdsRequest) FirestoreTestingHelper.assumeEnvVarSet(org.apache.beam.sdk.io.gcp.firestore.it.FirestoreTestingHelper.assumeEnvVarSet) FirestoreTestingHelper.chunkUpDocIds(org.apache.beam.sdk.io.gcp.firestore.it.FirestoreTestingHelper.chunkUpDocIds) Before(org.junit.Before) DoFn(org.apache.beam.sdk.transforms.DoFn) AssumptionViolatedException(org.junit.AssumptionViolatedException) MoreExecutors(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors) ApiFutures(com.google.api.core.ApiFutures) GcpOptions(org.apache.beam.sdk.extensions.gcp.options.GcpOptions) PAssert(org.apache.beam.sdk.testing.PAssert) Document(com.google.firestore.v1.Document) Test(org.junit.Test) UUID(java.util.UUID) PCollection(org.apache.beam.sdk.values.PCollection) Collectors(java.util.stream.Collectors) FirestoreIO(org.apache.beam.sdk.io.gcp.firestore.FirestoreIO) TimeUnit(java.util.concurrent.TimeUnit) ListDocumentsRequest(com.google.firestore.v1.ListDocumentsRequest) WriteBatch(com.google.cloud.firestore.WriteBatch) List(java.util.List) Rule(org.junit.Rule) Filter(org.apache.beam.sdk.transforms.Filter) ParDo(org.apache.beam.sdk.transforms.ParDo) DataLayout(org.apache.beam.sdk.io.gcp.firestore.it.FirestoreTestingHelper.DataLayout) Matchers.equalTo(org.hamcrest.Matchers.equalTo) RpcQosOptions(org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions) RunQueryResponse(com.google.firestore.v1.RunQueryResponse) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) PartitionQueryRequest(com.google.firestore.v1.PartitionQueryRequest) WriteBatch(com.google.cloud.firestore.WriteBatch) Test(org.junit.Test) TestDataLayoutHint(org.apache.beam.sdk.io.gcp.firestore.it.FirestoreTestingHelper.TestDataLayoutHint)

Example 2 with RpcQosOptions

use of org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions in project beam by apache.

the class FirestoreV1IT method batchWrite_partialFailureOutputsToDeadLetterQueue.

@Test
public void batchWrite_partialFailureOutputsToDeadLetterQueue() throws InterruptedException, ExecutionException, TimeoutException {
    String collectionId = "a";
    String docId = helper.docId();
    Write validWrite = Write.newBuilder().setUpdate(Document.newBuilder().setName(docPath(helper.getBaseDocumentPath(), collectionId, docId)).putFields("foo", Value.newBuilder().setStringValue(docId).build())).build();
    long millis = System.currentTimeMillis();
    Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000).setNanos((int) ((millis % 1000) * 1000000)).build();
    String docId2 = helper.docId();
    helper.getBaseDocument().collection(collectionId).document(docId2).create(ImmutableMap.of("foo", "baz")).get(10, TimeUnit.SECONDS);
    // this will fail because we're setting a updateTime precondition to before it was created
    Write conditionalUpdate = Write.newBuilder().setUpdate(Document.newBuilder().setName(docPath(helper.getBaseDocumentPath(), collectionId, docId2)).putFields("foo", Value.newBuilder().setStringValue(docId).build())).setCurrentDocument(Precondition.newBuilder().setUpdateTime(timestamp)).build();
    List<Write> writes = newArrayList(validWrite, conditionalUpdate);
    RpcQosOptions options = BaseFirestoreIT.rpcQosOptions.toBuilder().withBatchMaxCount(2).build();
    PCollection<WriteFailure> writeFailurePCollection = testPipeline.apply(Create.of(writes)).apply(FirestoreIO.v1().write().batchWrite().withDeadLetterQueue().withRpcQosOptions(options).build());
    PAssert.that(writeFailurePCollection).satisfies((writeFailures) -> {
        Iterator<WriteFailure> iterator = writeFailures.iterator();
        assertTrue(iterator.hasNext());
        WriteFailure failure = iterator.next();
        assertEquals(Code.FAILED_PRECONDITION, Code.forNumber(failure.getStatus().getCode()));
        assertNotNull(failure.getWriteResult());
        assertFalse(failure.getWriteResult().hasUpdateTime());
        assertEquals(conditionalUpdate, failure.getWrite());
        assertFalse(iterator.hasNext());
        return null;
    });
    testPipeline.run(this.options);
    ApiFuture<QuerySnapshot> actualDocsQuery = helper.getBaseDocument().collection(collectionId).orderBy("__name__").get();
    QuerySnapshot querySnapshot = actualDocsQuery.get(10, TimeUnit.SECONDS);
    List<QueryDocumentSnapshot> documents = querySnapshot.getDocuments();
    List<KV<String, String>> actualDocumentIds = documents.stream().map(doc -> KV.of(doc.getId(), doc.getString("foo"))).collect(Collectors.toList());
    List<KV<String, String>> expected = newArrayList(KV.of(docId, docId), KV.of(docId2, "baz"));
    assertEquals(expected, actualDocumentIds);
}
Also used : Write(com.google.firestore.v1.Write) KV(org.apache.beam.sdk.values.KV) RunQueryRequest(com.google.firestore.v1.RunQueryRequest) FieldReference(com.google.firestore.v1.StructuredQuery.FieldReference) Write(com.google.firestore.v1.Write) Direction(com.google.firestore.v1.StructuredQuery.Direction) TimeoutException(java.util.concurrent.TimeoutException) Operator(com.google.firestore.v1.StructuredQuery.FieldFilter.Operator) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) Timestamp(com.google.protobuf.Timestamp) QueryDocumentSnapshot(com.google.cloud.firestore.QueryDocumentSnapshot) BatchGetDocumentsRequest(com.google.firestore.v1.BatchGetDocumentsRequest) PTransform(org.apache.beam.sdk.transforms.PTransform) Lists.newArrayList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists.newArrayList) Value(com.google.firestore.v1.Value) Create(org.apache.beam.sdk.transforms.Create) QuerySnapshot(com.google.cloud.firestore.QuerySnapshot) WriteFailure(org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.WriteFailure) ListCollectionIdsRequest(com.google.firestore.v1.ListCollectionIdsRequest) Code(com.google.rpc.Code) DoFn(org.apache.beam.sdk.transforms.DoFn) Precondition(com.google.firestore.v1.Precondition) StructuredQuery(com.google.firestore.v1.StructuredQuery) Iterator(java.util.Iterator) PAssert(org.apache.beam.sdk.testing.PAssert) Assert.assertNotNull(org.junit.Assert.assertNotNull) Filter(com.google.firestore.v1.StructuredQuery.Filter) Document(com.google.firestore.v1.Document) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) PCollection(org.apache.beam.sdk.values.PCollection) Collectors(java.util.stream.Collectors) FirestoreIO(org.apache.beam.sdk.io.gcp.firestore.FirestoreIO) FieldFilter(com.google.firestore.v1.StructuredQuery.FieldFilter) ApiFuture(com.google.api.core.ApiFuture) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) ListDocumentsRequest(com.google.firestore.v1.ListDocumentsRequest) List(java.util.List) ParDo(org.apache.beam.sdk.transforms.ParDo) Assert.assertFalse(org.junit.Assert.assertFalse) RpcQosOptions(org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions) Order(com.google.firestore.v1.StructuredQuery.Order) CollectionSelector(com.google.firestore.v1.StructuredQuery.CollectionSelector) Assert.assertEquals(org.junit.Assert.assertEquals) PartitionQueryRequest(com.google.firestore.v1.PartitionQueryRequest) QueryDocumentSnapshot(com.google.cloud.firestore.QueryDocumentSnapshot) WriteFailure(org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.WriteFailure) KV(org.apache.beam.sdk.values.KV) Timestamp(com.google.protobuf.Timestamp) QuerySnapshot(com.google.cloud.firestore.QuerySnapshot) RpcQosOptions(org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions) Test(org.junit.Test)

Aggregations

BatchGetDocumentsRequest (com.google.firestore.v1.BatchGetDocumentsRequest)2 Document (com.google.firestore.v1.Document)2 ListCollectionIdsRequest (com.google.firestore.v1.ListCollectionIdsRequest)2 ListDocumentsRequest (com.google.firestore.v1.ListDocumentsRequest)2 PartitionQueryRequest (com.google.firestore.v1.PartitionQueryRequest)2 RunQueryRequest (com.google.firestore.v1.RunQueryRequest)2 Write (com.google.firestore.v1.Write)2 List (java.util.List)2 TimeUnit (java.util.concurrent.TimeUnit)2 Collectors (java.util.stream.Collectors)2 FirestoreIO (org.apache.beam.sdk.io.gcp.firestore.FirestoreIO)2 RpcQosOptions (org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions)2 PAssert (org.apache.beam.sdk.testing.PAssert)2 Create (org.apache.beam.sdk.transforms.Create)2 DoFn (org.apache.beam.sdk.transforms.DoFn)2 PTransform (org.apache.beam.sdk.transforms.PTransform)2 ParDo (org.apache.beam.sdk.transforms.ParDo)2 PCollection (org.apache.beam.sdk.values.PCollection)2 ImmutableMap (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap)2 Assert.assertEquals (org.junit.Assert.assertEquals)2