Examples with JsonStreamWriter com.google.cloud.bigquery.storage.v1.JsonStreamWriter used on opensource projects

Example 1 with JsonStreamWriter

use of com.google.cloud.bigquery.storage.v1.JsonStreamWriter in project java-bigquerystorage by googleapis.

the class ITBigQueryTimeEncoderTest method TestTimeEncoding.

@Test
public void TestTimeEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_STRING = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.STRING).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_str").build();
    TableFieldSchema TEST_TIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.TIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_time_micros").build();
    TableFieldSchema TEST_DATETIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATETIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_datetime_micros").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_STRING).addFields(1, TEST_TIME).addFields(2, TEST_DATETIME).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_str", "Start of the day");
        row.put("test_time_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(13, 14, 15, 16_000_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(23, 59, 59, 999_999_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(0, 0, 0, 0)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 2, 3, 4_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(5, 6, 7, 8_000)) }));
        row.put("test_datetime_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1, 1, 1, 12, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1995, 5, 19, 10, 30, 45, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2000, 1, 1, 0, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2026, 3, 11, 5, 45, 12, 9_000_000)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2050, 1, 2, 3, 4, 5, 6_000)) }));
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        Assert.assertFalse(response.get().getAppendResult().hasOffset());
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("Start of the day", currentRow.get(0).getValue());
        assertEquals("13:14:15.016000", currentRow.get(1).getRepeatedValue().get(0).getStringValue());
        assertEquals("23:59:59.999999", currentRow.get(1).getRepeatedValue().get(1).getStringValue());
        assertEquals("00:00:00", currentRow.get(1).getRepeatedValue().get(2).getStringValue());
        assertEquals("01:02:03.000004", currentRow.get(1).getRepeatedValue().get(3).getStringValue());
        assertEquals("05:06:07.000008", currentRow.get(1).getRepeatedValue().get(4).getStringValue());
        assertEquals("0001-01-01T12:00:00", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1995-05-19T10:30:45", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("2000-01-01T00:00:00", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("2026-03-11T05:45:12.009000", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("2050-01-02T03:04:05.000006", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 2 with JsonStreamWriter

use of com.google.cloud.bigquery.storage.v1.JsonStreamWriter in project java-bigquerystorage by googleapis.

the class ITBigQueryBigDecimalByteStringEncoderTest method TestBigDecimalEncoding.

@Test
public void TestBigDecimalEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_NUMERIC_ZERO = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_zero").build();
    TableFieldSchema TEST_NUMERIC_ONE = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_one").build();
    TableFieldSchema TEST_NUMERIC_REPEATED = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.REPEATED).setName("test_numeric_repeated").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_NUMERIC_ZERO).addFields(1, TEST_NUMERIC_ONE).addFields(2, TEST_NUMERIC_REPEATED).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_numeric_zero", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")));
        row.put("test_numeric_one", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")));
        row.put("test_numeric_repeated", new JSONArray(new byte[][] { BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("99999999999999999999999999999.999999999")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-99999999999999999999999999999.999999999")).toByteArray() }));
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        AppendRowsResponse arr = response.get();
        AppendResult ar = arr.getAppendResult();
        boolean ho = ar.hasOffset();
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("0", currentRow.get(0).getStringValue());
        assertEquals("1.2", currentRow.get(1).getStringValue());
        assertEquals("0", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1.2", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("-1.2", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("-99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 3 with JsonStreamWriter

use of com.google.cloud.bigquery.storage.v1.JsonStreamWriter in project java-bigquerystorage by googleapis.

the class ITBigQueryTimeEncoderTest method TestTimeEncoding.

@Test
public void TestTimeEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_STRING = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.STRING).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_str").build();
    TableFieldSchema TEST_TIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.TIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_time_micros").build();
    TableFieldSchema TEST_DATETIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATETIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_datetime_micros").build();
    TableFieldSchema TEST_DATE_REPEATED = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATE).setMode(TableFieldSchema.Mode.REPEATED).setName("test_date_repeated").build();
    TableFieldSchema TEST_DATE = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATE).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_date").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_STRING).addFields(1, TEST_TIME).addFields(2, TEST_DATETIME).addFields(3, TEST_DATE_REPEATED).addFields(4, TEST_DATE).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_str", "Start of the day");
        row.put("test_time_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(13, 14, 15, 16_000_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(23, 59, 59, 999_999_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(0, 0, 0, 0)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 2, 3, 4_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(5, 6, 7, 8_000)) }));
        row.put("test_datetime_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1, 1, 1, 12, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1995, 5, 19, 10, 30, 45, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2000, 1, 1, 0, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2026, 3, 11, 5, 45, 12, 9_000_000)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2050, 1, 2, 3, 4, 5, 6_000)) }));
        row.put("test_date_repeated", new JSONArray(new int[] { 0, 300, 14238 }));
        row.put("test_date", 300);
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        Assert.assertFalse(response.get().getAppendResult().hasOffset());
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("Start of the day", currentRow.get(0).getValue());
        assertEquals("13:14:15.016000", currentRow.get(1).getRepeatedValue().get(0).getStringValue());
        assertEquals("23:59:59.999999", currentRow.get(1).getRepeatedValue().get(1).getStringValue());
        assertEquals("00:00:00", currentRow.get(1).getRepeatedValue().get(2).getStringValue());
        assertEquals("01:02:03.000004", currentRow.get(1).getRepeatedValue().get(3).getStringValue());
        assertEquals("05:06:07.000008", currentRow.get(1).getRepeatedValue().get(4).getStringValue());
        assertEquals("0001-01-01T12:00:00", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1995-05-19T10:30:45", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("2000-01-01T00:00:00", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("1970-01-01", currentRow.get(3).getRepeatedValue().get(0).getStringValue());
        assertEquals("1970-10-28", currentRow.get(3).getRepeatedValue().get(1).getStringValue());
        assertEquals("2008-12-25", currentRow.get(3).getRepeatedValue().get(2).getStringValue());
        assertEquals("1970-10-28", currentRow.get(4).getStringValue());
        assertEquals("2026-03-11T05:45:12.009000", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("2050-01-02T03:04:05.000006", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 4 with JsonStreamWriter

use of com.google.cloud.bigquery.storage.v1.JsonStreamWriter in project java-bigquerystorage by googleapis.

the class ITBigQueryBigDecimalByteStringEncoderTest method TestBigDecimalEncoding.

@Test
public void TestBigDecimalEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_NUMERIC_ZERO = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_zero").build();
    TableFieldSchema TEST_NUMERIC_ONE = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_one").build();
    TableFieldSchema TEST_NUMERIC_REPEATED = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.REPEATED).setName("test_numeric_repeated").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_NUMERIC_ZERO).addFields(1, TEST_NUMERIC_ONE).addFields(2, TEST_NUMERIC_REPEATED).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_numeric_zero", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")));
        row.put("test_numeric_one", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")));
        row.put("test_numeric_repeated", new JSONArray(new byte[][] { BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("99999999999999999999999999999.999999999")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-99999999999999999999999999999.999999999")).toByteArray() }));
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        AppendRowsResponse arr = response.get();
        AppendResult ar = arr.getAppendResult();
        boolean ho = ar.hasOffset();
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("0", currentRow.get(0).getStringValue());
        assertEquals("1.2", currentRow.get(1).getStringValue());
        assertEquals("0", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1.2", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("-1.2", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("-99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 5 with JsonStreamWriter

use of com.google.cloud.bigquery.storage.v1.JsonStreamWriter in project java-bigquerystorage by googleapis.

the class WriteCommittedStream method writeCommittedStream.

public static void writeCommittedStream(String projectId, String datasetName, String tableName) throws DescriptorValidationException, InterruptedException, IOException {
    try (BigQueryWriteClient client = BigQueryWriteClient.create()) {
        // Initialize a write stream for the specified table.
        // For more information on WriteStream.Type, see:
        // https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/WriteStream.Type.html
        WriteStream stream = WriteStream.newBuilder().setType(WriteStream.Type.COMMITTED).build();
        TableName parentTable = TableName.of(projectId, datasetName, tableName);
        CreateWriteStreamRequest createWriteStreamRequest = CreateWriteStreamRequest.newBuilder().setParent(parentTable.toString()).setWriteStream(stream).build();
        WriteStream writeStream = client.createWriteStream(createWriteStreamRequest);
        // https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.html
        try (JsonStreamWriter writer = JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema()).build()) {
            // antipattern.
            for (int i = 0; i < 2; i++) {
                // Create a JSON object that is compatible with the table schema.
                JSONArray jsonArr = new JSONArray();
                for (int j = 0; j < 10; j++) {
                    JSONObject record = new JSONObject();
                    record.put("col1", String.format("record %03d-%03d", i, j));
                    jsonArr.put(record);
                }
                // To detect duplicate records, pass the index as the record offset.
                // To disable deduplication, omit the offset or use WriteStream.Type.DEFAULT.
                ApiFuture<AppendRowsResponse> future = writer.append(jsonArr, /*offset=*/
                i * 10);
                AppendRowsResponse response = future.get();
            }
            // Finalize the stream after use.
            FinalizeWriteStreamRequest finalizeWriteStreamRequest = FinalizeWriteStreamRequest.newBuilder().setName(writeStream.getName()).build();
            client.finalizeWriteStream(finalizeWriteStreamRequest);
        }
        System.out.println("Appended records successfully.");
    } catch (ExecutionException e) {
        // If the wrapped exception is a StatusRuntimeException, check the state of the operation.
        // If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
        // https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
        System.out.println("Failed to append records. \n" + e.toString());
    }
}