Search in sources :

Example 11 with AppendRowsResponse

use of com.google.cloud.bigquery.storage.v1.AppendRowsResponse in project java-bigquerystorage by googleapis.

the class AppendCompleteCallback method writeToDefaultStream.

// writeToDefaultStream: Writes records from the source file to the destination table.
public static void writeToDefaultStream(String projectId, String datasetName, String tableName, String dataFile) throws DescriptorValidationException, InterruptedException, IOException {
    BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
    // Get the schema of the destination table and convert to the equivalent BigQueryStorage type.
    Table table = bigquery.getTable(datasetName, tableName);
    Schema schema = table.getDefinition().getSchema();
    TableSchema tableSchema = BqToBqStorageSchemaConverter.convertTableSchema(schema);
    // Use the JSON stream writer to send records in JSON format.
    TableName parentTable = TableName.of(projectId, datasetName, tableName);
    try (JsonStreamWriter writer = JsonStreamWriter.newBuilder(parentTable.toString(), tableSchema).build()) {
        // Read JSON data from the source file and send it to the Write API.
        BufferedReader reader = new BufferedReader(new FileReader(dataFile));
        String line = reader.readLine();
        while (line != null) {
            // As a best practice, send batches of records, instead of single records at a time.
            JSONArray jsonArr = new JSONArray();
            for (int i = 0; i < 100; i++) {
                JSONObject record = new JSONObject(line);
                jsonArr.put(record);
                line = reader.readLine();
                if (line == null) {
                    break;
                }
            }
            // batch
            ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
            // The append method is asynchronous. Rather than waiting for the method to complete,
            // which can hurt performance, register a completion callback and continue streaming.
            ApiFutures.addCallback(future, new AppendCompleteCallback(), MoreExecutors.directExecutor());
        }
    }
}
Also used : BigQuery(com.google.cloud.bigquery.BigQuery) Table(com.google.cloud.bigquery.Table) TableSchema(com.google.cloud.bigquery.storage.v1.TableSchema) TableSchema(com.google.cloud.bigquery.storage.v1.TableSchema) Schema(com.google.cloud.bigquery.Schema) JSONArray(org.json.JSONArray) AppendRowsResponse(com.google.cloud.bigquery.storage.v1.AppendRowsResponse) TableName(com.google.cloud.bigquery.storage.v1.TableName) JSONObject(org.json.JSONObject) BufferedReader(java.io.BufferedReader) FileReader(java.io.FileReader) JsonStreamWriter(com.google.cloud.bigquery.storage.v1.JsonStreamWriter)

Example 12 with AppendRowsResponse

use of com.google.cloud.bigquery.storage.v1.AppendRowsResponse in project java-bigquerystorage by googleapis.

the class ParallelWriteCommittedStream method writeToStream.

private void writeToStream(BigQueryWriteClient client, WriteStream writeStream, long deadlineMillis) throws Throwable {
    LOG.info("Start writing to new stream:" + writeStream.getName());
    synchronized (this) {
        inflightCount = 0;
        successCount = 0;
        failureCount = 0;
        error = null;
        lastMetricsTimeMillis = System.currentTimeMillis();
        lastMetricsSuccessCount = 0;
        lastMetricsFailureCount = 0;
    }
    Descriptor descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(writeStream.getTableSchema());
    ProtoSchema protoSchema = ProtoSchemaConverter.convert(descriptor);
    try (StreamWriter writer = StreamWriter.newBuilder(writeStream.getName()).setWriterSchema(protoSchema).setTraceId("SAMPLE:parallel_append").build()) {
        while (System.currentTimeMillis() < deadlineMillis) {
            synchronized (this) {
                if (error != null) {
                    // Stop writing once we get an error.
                    throw error;
                }
            }
            ApiFuture<AppendRowsResponse> future = writer.append(createAppendRows(descriptor), -1);
            synchronized (this) {
                inflightCount++;
            }
            ApiFutures.addCallback(future, new AppendCompleteCallback(this), MoreExecutors.directExecutor());
        }
    }
}
Also used : ProtoSchema(com.google.cloud.bigquery.storage.v1.ProtoSchema) StreamWriter(com.google.cloud.bigquery.storage.v1.StreamWriter) AppendRowsResponse(com.google.cloud.bigquery.storage.v1.AppendRowsResponse) Descriptor(com.google.protobuf.Descriptors.Descriptor) BQTableSchemaToProtoDescriptor(com.google.cloud.bigquery.storage.v1.BQTableSchemaToProtoDescriptor)

Example 13 with AppendRowsResponse

use of com.google.cloud.bigquery.storage.v1.AppendRowsResponse in project java-bigquerystorage by googleapis.

the class WriteBufferedStream method writeBufferedStream.

public static void writeBufferedStream(String projectId, String datasetName, String tableName) throws DescriptorValidationException, InterruptedException, IOException {
    try (BigQueryWriteClient client = BigQueryWriteClient.create()) {
        // Initialize a write stream for the specified table.
        // For more information on WriteStream.Type, see:
        // https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1/WriteStream.Type.html
        WriteStream stream = WriteStream.newBuilder().setType(WriteStream.Type.BUFFERED).build();
        TableName parentTable = TableName.of(projectId, datasetName, tableName);
        CreateWriteStreamRequest createWriteStreamRequest = CreateWriteStreamRequest.newBuilder().setParent(parentTable.toString()).setWriteStream(stream).build();
        WriteStream writeStream = client.createWriteStream(createWriteStreamRequest);
        // https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.html
        try (JsonStreamWriter writer = JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema()).build()) {
            // Write two batches to the stream, each with 10 JSON records.
            for (int i = 0; i < 2; i++) {
                JSONArray jsonArr = new JSONArray();
                for (int j = 0; j < 10; j++) {
                    // Create a JSON object that is compatible with the table schema.
                    JSONObject record = new JSONObject();
                    record.put("col1", String.format("buffered-record %03d", i));
                    jsonArr.put(record);
                }
                ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
                AppendRowsResponse response = future.get();
            }
            // Flush the buffer.
            FlushRowsRequest flushRowsRequest = FlushRowsRequest.newBuilder().setWriteStream(writeStream.getName()).setOffset(// Advance the cursor to the latest record.
            Int64Value.of(10 * 2 - 1)).build();
            FlushRowsResponse flushRowsResponse = client.flushRows(flushRowsRequest);
        // You can continue to write to the stream after flushing the buffer.
        }
        // Finalize the stream after use.
        FinalizeWriteStreamRequest finalizeWriteStreamRequest = FinalizeWriteStreamRequest.newBuilder().setName(writeStream.getName()).build();
        client.finalizeWriteStream(finalizeWriteStreamRequest);
        System.out.println("Appended and committed records successfully.");
    } catch (ExecutionException e) {
        // If the wrapped exception is a StatusRuntimeException, check the state of the operation.
        // If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
        // https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
        System.out.println(e);
    }
}
Also used : FinalizeWriteStreamRequest(com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamRequest) JSONArray(org.json.JSONArray) AppendRowsResponse(com.google.cloud.bigquery.storage.v1.AppendRowsResponse) WriteStream(com.google.cloud.bigquery.storage.v1.WriteStream) BigQueryWriteClient(com.google.cloud.bigquery.storage.v1.BigQueryWriteClient) TableName(com.google.cloud.bigquery.storage.v1.TableName) CreateWriteStreamRequest(com.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest) JSONObject(org.json.JSONObject) ExecutionException(java.util.concurrent.ExecutionException) JsonStreamWriter(com.google.cloud.bigquery.storage.v1.JsonStreamWriter) FlushRowsRequest(com.google.cloud.bigquery.storage.v1.FlushRowsRequest) FlushRowsResponse(com.google.cloud.bigquery.storage.v1.FlushRowsResponse)

Aggregations

JSONArray (org.json.JSONArray)9 JSONObject (org.json.JSONObject)9 AppendRowsResponse (com.google.cloud.bigquery.storage.v1.AppendRowsResponse)8 JsonStreamWriter (com.google.cloud.bigquery.storage.v1.JsonStreamWriter)5 TableName (com.google.cloud.bigquery.storage.v1.TableName)5 ExecutionException (java.util.concurrent.ExecutionException)5 FieldValueList (com.google.cloud.bigquery.FieldValueList)4 TableResult (com.google.cloud.bigquery.TableResult)4 WriteStream (com.google.cloud.bigquery.storage.v1.WriteStream)4 AppendRowsResponse (com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse)4 JsonStreamWriter (com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter)4 TableFieldSchema (com.google.cloud.bigquery.storage.v1beta2.TableFieldSchema)4 TableName (com.google.cloud.bigquery.storage.v1beta2.TableName)4 TableSchema (com.google.cloud.bigquery.storage.v1beta2.TableSchema)4 Test (org.junit.Test)4 BigQueryWriteClient (com.google.cloud.bigquery.storage.v1.BigQueryWriteClient)3 CreateWriteStreamRequest (com.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest)3 BigQuery (com.google.cloud.bigquery.BigQuery)2 Schema (com.google.cloud.bigquery.Schema)2 Table (com.google.cloud.bigquery.Table)2