Examples with CreateReadSessionRequest com.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest used on opensource projects

Example 16 with CreateReadSessionRequest

use of com.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest in project beam by apache.

the class BigQueryIOStorageReadTest method testReadFromBigQueryIOWithTrimmedSchema.

@Test
public void testReadFromBigQueryIOWithTrimmedSchema() throws Exception {
    fakeDatasetService.createDataset("foo.com:project", "dataset", "", "", null);
    TableReference tableRef = BigQueryHelpers.parseTableSpec("foo.com:project:dataset.table");
    Table table = new Table().setTableReference(tableRef).setNumBytes(10L).setSchema(TABLE_SCHEMA);
    fakeDatasetService.createTable(table);
    CreateReadSessionRequest expectedCreateReadSessionRequest = CreateReadSessionRequest.newBuilder().setParent("projects/project-id").setReadSession(ReadSession.newBuilder().setTable("projects/foo.com:project/datasets/dataset/tables/table").setReadOptions(ReadSession.TableReadOptions.newBuilder().addSelectedFields("name")).setDataFormat(DataFormat.AVRO)).setMaxStreamCount(10).build();
    ReadSession readSession = ReadSession.newBuilder().setName("readSessionName").setAvroSchema(AvroSchema.newBuilder().setSchema(TRIMMED_AVRO_SCHEMA_STRING)).addStreams(ReadStream.newBuilder().setName("streamName")).setDataFormat(DataFormat.AVRO).build();
    ReadRowsRequest expectedReadRowsRequest = ReadRowsRequest.newBuilder().setReadStream("streamName").build();
    List<GenericRecord> records = Lists.newArrayList(createRecord("A", TRIMMED_AVRO_SCHEMA), createRecord("B", TRIMMED_AVRO_SCHEMA), createRecord("C", TRIMMED_AVRO_SCHEMA), createRecord("D", TRIMMED_AVRO_SCHEMA));
    List<ReadRowsResponse> readRowsResponses = Lists.newArrayList(createResponse(TRIMMED_AVRO_SCHEMA, records.subList(0, 2), 0.0, 0.50), createResponse(TRIMMED_AVRO_SCHEMA, records.subList(2, 4), 0.5, 0.75));
    StorageClient fakeStorageClient = mock(StorageClient.class, withSettings().serializable());
    when(fakeStorageClient.createReadSession(expectedCreateReadSessionRequest)).thenReturn(readSession);
    when(fakeStorageClient.readRows(expectedReadRowsRequest, "")).thenReturn(new FakeBigQueryServerStream<>(readRowsResponses));
    PCollection<TableRow> output = p.apply(BigQueryIO.readTableRows().from("foo.com:project:dataset.table").withMethod(Method.DIRECT_READ).withSelectedFields(Lists.newArrayList("name")).withFormat(DataFormat.AVRO).withTestServices(new FakeBigQueryServices().withDatasetService(fakeDatasetService).withStorageClient(fakeStorageClient)));
    PAssert.that(output).containsInAnyOrder(ImmutableList.of(new TableRow().set("name", "A"), new TableRow().set("name", "B"), new TableRow().set("name", "C"), new TableRow().set("name", "D")));
    p.run();
}

Example 17 with CreateReadSessionRequest

use of com.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest in project beam by apache.

the class BigQueryServicesImplTest method testCreateReadSessionSetsRequestCountMetric.

@Test
public void testCreateReadSessionSetsRequestCountMetric() throws InterruptedException, IOException {
    BigQueryServicesImpl.StorageClientImpl client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    CreateReadSessionRequest.Builder builder = CreateReadSessionRequest.newBuilder();
    builder.getReadSessionBuilder().setTable("myproject:mydataset.mytable");
    CreateReadSessionRequest request = builder.build();
    when(client.callCreateReadSession(request)).thenReturn(// Mock implementation.
    ReadSession.newBuilder().build());
    // Real implementation.
    when(client.createReadSession(any())).thenCallRealMethod();
    client.createReadSession(request);
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "ok", 1);
}

Example 18 with CreateReadSessionRequest

use of com.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest in project beam by apache.

the class BigQueryServicesImplTest method testCreateReadSessionSetsRequestCountMetricOnError.

@Test
public void testCreateReadSessionSetsRequestCountMetricOnError() throws InterruptedException, IOException {
    BigQueryServicesImpl.StorageClientImpl client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    CreateReadSessionRequest.Builder builder = CreateReadSessionRequest.newBuilder();
    builder.getReadSessionBuilder().setTable("myproject:mydataset.mytable");
    CreateReadSessionRequest request = builder.build();
    StatusCode statusCode = new StatusCode() {

        @Override
        public Code getCode() {
            return Code.NOT_FOUND;
        }

        @Override
        public Object getTransportCode() {
            return null;
        }
    };
    when(client.callCreateReadSession(request)).thenThrow(// Mock implementation.
    new ApiException("Not Found", null, statusCode, false));
    // Real implementation.
    when(client.createReadSession(any())).thenCallRealMethod();
    thrown.expect(ApiException.class);
    thrown.expectMessage("Not Found");
    client.createReadSession(request);
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "not_found", 1);
}

Example 19 with CreateReadSessionRequest

use of com.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest in project spark-bigquery-connector by GoogleCloudDataproc.

the class ReadSessionCreatorTest method testSerializedInstanceIsPropagated.

@Test
public void testSerializedInstanceIsPropagated() throws Exception {
    TableReadOptions tableReadOptions = TableReadOptions.newBuilder().build();
    ReadSession readSession = ReadSession.newBuilder().setName("abc").setReadOptions(tableReadOptions).build();
    CreateReadSessionRequest request = CreateReadSessionRequest.newBuilder().setReadSession(readSession).build();
    Optional<String> encodedBase = Optional.of(java.util.Base64.getEncoder().encodeToString(request.toByteArray()));
    ReadSessionCreatorConfig config = new ReadSessionCreatorConfigBuilder().setRequestEncodedBase(encodedBase).build();
    ReadSessionCreator creator = new ReadSessionCreator(config, bigQueryClient, bigQueryReadClientFactory);
    when(bigQueryReadClientFactory.getBigQueryReadClient()).thenReturn(readClient);
    when(bigQueryClient.getTable(any())).thenReturn(table);
    when(stub.createReadSessionCallable()).thenReturn(createReadSessionCall);
    creator.create(TableId.of("dataset", "table"), ImmutableList.of("col1", "col2"), Optional.empty()).getReadSession();
    ArgumentCaptor<CreateReadSessionRequest> requestCaptor = ArgumentCaptor.forClass(CreateReadSessionRequest.class);
    verify(createReadSessionCall, times(1)).call(requestCaptor.capture());
    ReadSession actual = requestCaptor.getValue().getReadSession();
    assertThat(actual.getName()).isEqualTo("abc");
    assertThat(actual.getReadOptions().getSelectedFieldsList()).containsExactly("col1", "col2");
}

Example 20 with CreateReadSessionRequest

use of com.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest in project java-bigquerystorage by googleapis.

the class ITBigQueryStorageTest method testFilter.

@Test
public void testFilter() throws IOException {
    String table = BigQueryResource.FormatTableResource(/* projectId = */
    "bigquery-public-data", /* datasetId = */
    "samples", /* tableId = */
    "shakespeare");
    TableReadOptions options = TableReadOptions.newBuilder().setRowRestriction("word_count > 100").build();
    CreateReadSessionRequest request = CreateReadSessionRequest.newBuilder().setParent(parentProjectId).setMaxStreamCount(1).setReadSession(ReadSession.newBuilder().setTable(table).setReadOptions(options).setDataFormat(DataFormat.AVRO).build()).build();
    ReadSession session = client.createReadSession(request);
    assertEquals(String.format("Did not receive expected number of streams for table '%s' CreateReadSession response:%n%s", table, session.toString()), 1, session.getStreamsCount());
    ReadRowsRequest readRowsRequest = ReadRowsRequest.newBuilder().setReadStream(session.getStreams(0).getName()).build();
    SimpleRowReader reader = new SimpleRowReader(new Schema.Parser().parse(session.getAvroSchema().getSchema()));
    long rowCount = 0;
    ServerStream<ReadRowsResponse> stream = client.readRowsCallable().call(readRowsRequest);
    for (ReadRowsResponse response : stream) {
        rowCount += response.getRowCount();
        reader.processRows(response.getAvroRows(), new AvroRowConsumer() {

            @Override
            public void accept(GenericData.Record record) {
                Long wordCount = (Long) record.get("word_count");
                assertWithMessage("Row not matching expectations: %s", record.toString()).that(wordCount).isGreaterThan(100L);
            }
        });
    }
    assertEquals(1_333, rowCount);
}