Search in sources :

Example 31 with TableName

use of com.google.cloud.bigquery.storage.v1beta2.TableName in project java-bigtable by googleapis.

the class BaseBigtableTableAdminClientTest method deleteTableTest.

@Test
public void deleteTableTest() throws Exception {
    Empty expectedResponse = Empty.newBuilder().build();
    mockBigtableTableAdmin.addResponse(expectedResponse);
    TableName name = TableName.of("[PROJECT]", "[INSTANCE]", "[TABLE]");
    client.deleteTable(name);
    List<AbstractMessage> actualRequests = mockBigtableTableAdmin.getRequests();
    Assert.assertEquals(1, actualRequests.size());
    DeleteTableRequest actualRequest = ((DeleteTableRequest) actualRequests.get(0));
    Assert.assertEquals(name.toString(), actualRequest.getName());
    Assert.assertTrue(channelProvider.isHeaderSent(ApiClientHeaderProvider.getDefaultApiClientHeaderKey(), GaxGrpcProperties.getDefaultApiClientHeaderPattern()));
}
Also used : DeleteTableRequest(com.google.bigtable.admin.v2.DeleteTableRequest) TableName(com.google.bigtable.admin.v2.TableName) Empty(com.google.protobuf.Empty) AbstractMessage(com.google.protobuf.AbstractMessage) Test(org.junit.Test)

Example 32 with TableName

use of com.google.cloud.bigquery.storage.v1beta2.TableName in project java-bigtable by googleapis.

the class BaseBigtableTableAdminClientTest method generateConsistencyTokenTest.

@Test
public void generateConsistencyTokenTest() throws Exception {
    GenerateConsistencyTokenResponse expectedResponse = GenerateConsistencyTokenResponse.newBuilder().setConsistencyToken("consistencyToken-1985152319").build();
    mockBigtableTableAdmin.addResponse(expectedResponse);
    TableName name = TableName.of("[PROJECT]", "[INSTANCE]", "[TABLE]");
    GenerateConsistencyTokenResponse actualResponse = client.generateConsistencyToken(name);
    Assert.assertEquals(expectedResponse, actualResponse);
    List<AbstractMessage> actualRequests = mockBigtableTableAdmin.getRequests();
    Assert.assertEquals(1, actualRequests.size());
    GenerateConsistencyTokenRequest actualRequest = ((GenerateConsistencyTokenRequest) actualRequests.get(0));
    Assert.assertEquals(name.toString(), actualRequest.getName());
    Assert.assertTrue(channelProvider.isHeaderSent(ApiClientHeaderProvider.getDefaultApiClientHeaderKey(), GaxGrpcProperties.getDefaultApiClientHeaderPattern()));
}
Also used : TableName(com.google.bigtable.admin.v2.TableName) AbstractMessage(com.google.protobuf.AbstractMessage) GenerateConsistencyTokenRequest(com.google.bigtable.admin.v2.GenerateConsistencyTokenRequest) GenerateConsistencyTokenResponse(com.google.bigtable.admin.v2.GenerateConsistencyTokenResponse) Test(org.junit.Test)

Example 33 with TableName

use of com.google.cloud.bigquery.storage.v1beta2.TableName in project java-bigtable by googleapis.

the class BaseBigtableTableAdminClientTest method snapshotTableExceptionTest2.

@Test
public void snapshotTableExceptionTest2() throws Exception {
    StatusRuntimeException exception = new StatusRuntimeException(io.grpc.Status.INVALID_ARGUMENT);
    mockBigtableTableAdmin.addException(exception);
    try {
        TableName name = TableName.of("[PROJECT]", "[INSTANCE]", "[TABLE]");
        String cluster = "cluster872092154";
        String snapshotId = "snapshotId-1113817601";
        String description = "description-1724546052";
        client.snapshotTableAsync(name, cluster, snapshotId, description).get();
        Assert.fail("No exception raised");
    } catch (ExecutionException e) {
        Assert.assertEquals(InvalidArgumentException.class, e.getCause().getClass());
        InvalidArgumentException apiException = ((InvalidArgumentException) e.getCause());
        Assert.assertEquals(StatusCode.Code.INVALID_ARGUMENT, apiException.getStatusCode().getCode());
    }
}
Also used : TableName(com.google.bigtable.admin.v2.TableName) InvalidArgumentException(com.google.api.gax.rpc.InvalidArgumentException) StatusRuntimeException(io.grpc.StatusRuntimeException) ByteString(com.google.protobuf.ByteString) ExecutionException(java.util.concurrent.ExecutionException) Test(org.junit.Test)

Example 34 with TableName

use of com.google.cloud.bigquery.storage.v1beta2.TableName in project java-bigtable by googleapis.

the class BaseBigtableTableAdminClientTest method deleteTableExceptionTest.

@Test
public void deleteTableExceptionTest() throws Exception {
    StatusRuntimeException exception = new StatusRuntimeException(io.grpc.Status.INVALID_ARGUMENT);
    mockBigtableTableAdmin.addException(exception);
    try {
        TableName name = TableName.of("[PROJECT]", "[INSTANCE]", "[TABLE]");
        client.deleteTable(name);
        Assert.fail("No exception raised");
    } catch (InvalidArgumentException e) {
    // Expected exception.
    }
}
Also used : TableName(com.google.bigtable.admin.v2.TableName) InvalidArgumentException(com.google.api.gax.rpc.InvalidArgumentException) StatusRuntimeException(io.grpc.StatusRuntimeException) Test(org.junit.Test)

Example 35 with TableName

use of com.google.cloud.bigquery.storage.v1beta2.TableName in project dataproc-templates by GoogleCloudPlatform.

the class PubSubToBQ method writeToBQ.

public static void writeToBQ(JavaDStream<SparkPubsubMessage> pubSubStream, String outputProjectID, String pubSubBQOutputDataset, String PubSubBQOutputTable, Integer batchSize) {
    pubSubStream.foreachRDD(new VoidFunction<JavaRDD<SparkPubsubMessage>>() {

        @Override
        public void call(JavaRDD<SparkPubsubMessage> sparkPubsubMessageJavaRDD) throws Exception {
            sparkPubsubMessageJavaRDD.foreachPartition(new VoidFunction<Iterator<SparkPubsubMessage>>() {

                @Override
                public void call(Iterator<SparkPubsubMessage> sparkPubsubMessageIterator) throws Exception {
                    BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
                    Table table = bigquery.getTable(pubSubBQOutputDataset, PubSubBQOutputTable);
                    TableName parentTable = TableName.of(outputProjectID, pubSubBQOutputDataset, PubSubBQOutputTable);
                    Schema schema = table.getDefinition().getSchema();
                    JsonStreamWriter writer = JsonStreamWriter.newBuilder(parentTable.toString(), schema).build();
                    JSONArray jsonArr = new JSONArray();
                    while (sparkPubsubMessageIterator.hasNext()) {
                        SparkPubsubMessage message = sparkPubsubMessageIterator.next();
                        JSONObject record = new JSONObject(new String(message.getData()));
                        jsonArr.put(record);
                        if (jsonArr.length() == batchSize) {
                            ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
                            AppendRowsResponse response = future.get();
                            jsonArr = new JSONArray();
                        }
                    }
                    if (jsonArr.length() > 0) {
                        ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
                        AppendRowsResponse response = future.get();
                    }
                    writer.close();
                }
            });
        }
    });
}
Also used : JSONArray(com.google.cloud.spark.bigquery.repackaged.org.json.JSONArray) AppendRowsResponse(com.google.cloud.spark.bigquery.repackaged.com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse) JavaRDD(org.apache.spark.api.java.JavaRDD) TableName(com.google.cloud.spark.bigquery.repackaged.com.google.cloud.bigquery.storage.v1beta2.TableName) SparkPubsubMessage(org.apache.spark.streaming.pubsub.SparkPubsubMessage) JSONObject(com.google.cloud.spark.bigquery.repackaged.org.json.JSONObject) VoidFunction(org.apache.spark.api.java.function.VoidFunction) Iterator(java.util.Iterator) JsonStreamWriter(com.google.cloud.spark.bigquery.repackaged.com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter)

Aggregations

Test (org.junit.Test)33 ByteString (com.google.protobuf.ByteString)28 ArrayList (java.util.ArrayList)20 TableName (com.google.bigtable.v2.TableName)18 TableName (com.google.bigtable.admin.v2.TableName)17 InvalidArgumentException (com.google.api.gax.rpc.InvalidArgumentException)13 AbstractMessage (com.google.protobuf.AbstractMessage)13 StatusRuntimeException (io.grpc.StatusRuntimeException)13 Mutation (com.google.bigtable.v2.Mutation)12 JSONArray (org.json.JSONArray)9 JSONObject (org.json.JSONObject)9 ExecutionException (java.util.concurrent.ExecutionException)8 TableName (com.google.cloud.bigquery.storage.v1.TableName)7 ReadModifyWriteRule (com.google.bigtable.v2.ReadModifyWriteRule)6 RowFilter (com.google.bigtable.v2.RowFilter)6 BaseBigtableDataClient (com.google.cloud.bigtable.data.v2.BaseBigtableDataClient)6 ColumnFamily (com.google.bigtable.admin.v2.ColumnFamily)5 CheckAndMutateRowResponse (com.google.bigtable.v2.CheckAndMutateRowResponse)4 ReadModifyWriteRowResponse (com.google.bigtable.v2.ReadModifyWriteRowResponse)4 FieldValueList (com.google.cloud.bigquery.FieldValueList)4