Examples with TableSchema com.google.cloud.bigquery.storage.v1.TableSchema used on opensource projects

Example 1 with TableSchema

use of com.google.cloud.bigquery.storage.v1.TableSchema in project java-bigquerystorage by googleapis.

the class ITBigQueryTimeEncoderTest method TestTimeEncoding.

@Test
public void TestTimeEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_STRING = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.STRING).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_str").build();
    TableFieldSchema TEST_TIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.TIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_time_micros").build();
    TableFieldSchema TEST_DATETIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATETIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_datetime_micros").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_STRING).addFields(1, TEST_TIME).addFields(2, TEST_DATETIME).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_str", "Start of the day");
        row.put("test_time_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(13, 14, 15, 16_000_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(23, 59, 59, 999_999_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(0, 0, 0, 0)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 2, 3, 4_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(5, 6, 7, 8_000)) }));
        row.put("test_datetime_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1, 1, 1, 12, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1995, 5, 19, 10, 30, 45, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2000, 1, 1, 0, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2026, 3, 11, 5, 45, 12, 9_000_000)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2050, 1, 2, 3, 4, 5, 6_000)) }));
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        Assert.assertFalse(response.get().getAppendResult().hasOffset());
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("Start of the day", currentRow.get(0).getValue());
        assertEquals("13:14:15.016000", currentRow.get(1).getRepeatedValue().get(0).getStringValue());
        assertEquals("23:59:59.999999", currentRow.get(1).getRepeatedValue().get(1).getStringValue());
        assertEquals("00:00:00", currentRow.get(1).getRepeatedValue().get(2).getStringValue());
        assertEquals("01:02:03.000004", currentRow.get(1).getRepeatedValue().get(3).getStringValue());
        assertEquals("05:06:07.000008", currentRow.get(1).getRepeatedValue().get(4).getStringValue());
        assertEquals("0001-01-01T12:00:00", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1995-05-19T10:30:45", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("2000-01-01T00:00:00", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("2026-03-11T05:45:12.009000", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("2050-01-02T03:04:05.000006", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 2 with TableSchema

use of com.google.cloud.bigquery.storage.v1.TableSchema in project java-bigquerystorage by googleapis.

the class ITBigQueryBigDecimalByteStringEncoderTest method TestBigDecimalEncoding.

@Test
public void TestBigDecimalEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_NUMERIC_ZERO = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_zero").build();
    TableFieldSchema TEST_NUMERIC_ONE = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_one").build();
    TableFieldSchema TEST_NUMERIC_REPEATED = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.REPEATED).setName("test_numeric_repeated").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_NUMERIC_ZERO).addFields(1, TEST_NUMERIC_ONE).addFields(2, TEST_NUMERIC_REPEATED).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_numeric_zero", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")));
        row.put("test_numeric_one", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")));
        row.put("test_numeric_repeated", new JSONArray(new byte[][] { BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("99999999999999999999999999999.999999999")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-99999999999999999999999999999.999999999")).toByteArray() }));
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        AppendRowsResponse arr = response.get();
        AppendResult ar = arr.getAppendResult();
        boolean ho = ar.hasOffset();
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("0", currentRow.get(0).getStringValue());
        assertEquals("1.2", currentRow.get(1).getStringValue());
        assertEquals("0", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1.2", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("-1.2", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("-99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 3 with TableSchema

use of com.google.cloud.bigquery.storage.v1.TableSchema in project java-bigquerystorage by googleapis.

the class ITBigQueryTimeEncoderTest method TestTimeEncoding.

@Test
public void TestTimeEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_STRING = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.STRING).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_str").build();
    TableFieldSchema TEST_TIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.TIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_time_micros").build();
    TableFieldSchema TEST_DATETIME = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATETIME).setMode(TableFieldSchema.Mode.REPEATED).setName("test_datetime_micros").build();
    TableFieldSchema TEST_DATE_REPEATED = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATE).setMode(TableFieldSchema.Mode.REPEATED).setName("test_date_repeated").build();
    TableFieldSchema TEST_DATE = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.DATE).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_date").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_STRING).addFields(1, TEST_TIME).addFields(2, TEST_DATETIME).addFields(3, TEST_DATE_REPEATED).addFields(4, TEST_DATE).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_str", "Start of the day");
        row.put("test_time_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(13, 14, 15, 16_000_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(23, 59, 59, 999_999_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(0, 0, 0, 0)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 2, 3, 4_000)), CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(5, 6, 7, 8_000)) }));
        row.put("test_datetime_micros", new JSONArray(new long[] { CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1, 1, 1, 12, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(1995, 5, 19, 10, 30, 45, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2000, 1, 1, 0, 0, 0, 0)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2026, 3, 11, 5, 45, 12, 9_000_000)), CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2050, 1, 2, 3, 4, 5, 6_000)) }));
        row.put("test_date_repeated", new JSONArray(new int[] { 0, 300, 14238 }));
        row.put("test_date", 300);
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        Assert.assertFalse(response.get().getAppendResult().hasOffset());
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("Start of the day", currentRow.get(0).getValue());
        assertEquals("13:14:15.016000", currentRow.get(1).getRepeatedValue().get(0).getStringValue());
        assertEquals("23:59:59.999999", currentRow.get(1).getRepeatedValue().get(1).getStringValue());
        assertEquals("00:00:00", currentRow.get(1).getRepeatedValue().get(2).getStringValue());
        assertEquals("01:02:03.000004", currentRow.get(1).getRepeatedValue().get(3).getStringValue());
        assertEquals("05:06:07.000008", currentRow.get(1).getRepeatedValue().get(4).getStringValue());
        assertEquals("0001-01-01T12:00:00", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1995-05-19T10:30:45", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("2000-01-01T00:00:00", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("1970-01-01", currentRow.get(3).getRepeatedValue().get(0).getStringValue());
        assertEquals("1970-10-28", currentRow.get(3).getRepeatedValue().get(1).getStringValue());
        assertEquals("2008-12-25", currentRow.get(3).getRepeatedValue().get(2).getStringValue());
        assertEquals("1970-10-28", currentRow.get(4).getStringValue());
        assertEquals("2026-03-11T05:45:12.009000", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("2050-01-02T03:04:05.000006", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 4 with TableSchema

use of com.google.cloud.bigquery.storage.v1.TableSchema in project java-bigquerystorage by googleapis.

the class ITBigQueryBigDecimalByteStringEncoderTest method TestBigDecimalEncoding.

@Test
public void TestBigDecimalEncoding() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException {
    TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, TABLE);
    TableFieldSchema TEST_NUMERIC_ZERO = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_zero").build();
    TableFieldSchema TEST_NUMERIC_ONE = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.NULLABLE).setName("test_numeric_one").build();
    TableFieldSchema TEST_NUMERIC_REPEATED = TableFieldSchema.newBuilder().setType(TableFieldSchema.Type.NUMERIC).setMode(TableFieldSchema.Mode.REPEATED).setName("test_numeric_repeated").build();
    TableSchema tableSchema = TableSchema.newBuilder().addFields(0, TEST_NUMERIC_ZERO).addFields(1, TEST_NUMERIC_ONE).addFields(2, TEST_NUMERIC_REPEATED).build();
    try (JsonStreamWriter jsonStreamWriter = JsonStreamWriter.newBuilder(parent.toString(), tableSchema).build()) {
        JSONObject row = new JSONObject();
        row.put("test_numeric_zero", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")));
        row.put("test_numeric_one", BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")));
        row.put("test_numeric_repeated", new JSONArray(new byte[][] { BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-1.2")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("99999999999999999999999999999.999999999")).toByteArray(), BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-99999999999999999999999999999.999999999")).toByteArray() }));
        JSONArray jsonArr = new JSONArray(new JSONObject[] { row });
        ApiFuture<AppendRowsResponse> response = jsonStreamWriter.append(jsonArr, -1);
        AppendRowsResponse arr = response.get();
        AppendResult ar = arr.getAppendResult();
        boolean ho = ar.hasOffset();
        TableResult result = bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
        Iterator<FieldValueList> iter = result.getValues().iterator();
        FieldValueList currentRow;
        currentRow = iter.next();
        assertEquals("0", currentRow.get(0).getStringValue());
        assertEquals("1.2", currentRow.get(1).getStringValue());
        assertEquals("0", currentRow.get(2).getRepeatedValue().get(0).getStringValue());
        assertEquals("1.2", currentRow.get(2).getRepeatedValue().get(1).getStringValue());
        assertEquals("-1.2", currentRow.get(2).getRepeatedValue().get(2).getStringValue());
        assertEquals("99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(3).getStringValue());
        assertEquals("-99999999999999999999999999999.999999999", currentRow.get(2).getRepeatedValue().get(4).getStringValue());
    }
}

Example 5 with TableSchema

use of com.google.cloud.bigquery.storage.v1.TableSchema in project java-bigquerystorage by googleapis.

the class WriteToDefaultStream method writeToDefaultStream.

public static void writeToDefaultStream(String projectId, String datasetName, String tableName) throws DescriptorValidationException, InterruptedException, IOException {
    BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
    Table table = bigquery.getTable(datasetName, tableName);
    TableName parentTable = TableName.of(projectId, datasetName, tableName);
    Schema schema = table.getDefinition().getSchema();
    TableSchema tableSchema = BqToBqStorageSchemaConverter.convertTableSchema(schema);
    // https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.html
    try (JsonStreamWriter writer = JsonStreamWriter.newBuilder(parentTable.toString(), tableSchema).build()) {
        // much writes as possible. Creating a writer for just one write is an antipattern.
        for (int i = 0; i < 2; i++) {
            // Create a JSON object that is compatible with the table schema.
            JSONArray jsonArr = new JSONArray();
            for (int j = 0; j < 10; j++) {
                JSONObject record = new JSONObject();
                record.put("test_string", String.format("record %03d-%03d", i, j));
                jsonArr.put(record);
            }
            ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
            AppendRowsResponse response = future.get();
        }
        System.out.println("Appended records successfully.");
    } catch (ExecutionException e) {
        // If the wrapped exception is a StatusRuntimeException, check the state of the operation.
        // If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
        // https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
        System.out.println("Failed to append records. \n" + e.toString());
    }
}