Examples with ReadStream com.google.cloud.bigquery.storage.v1beta2.ReadStream used on opensource projects

Example 1 with ReadStream

use of com.google.cloud.bigquery.storage.v1beta2.ReadStream in project java-bigquerystorage by googleapis.

the class ITBigQueryStorageLongRunningTest method readAllRowsFromStream.

private long readAllRowsFromStream(ReadStream readStream) {
    ReadRowsRequest readRowsRequest = ReadRowsRequest.newBuilder().setReadStream(readStream.getName()).build();
    long rowCount = 0;
    ServerStream<ReadRowsResponse> serverStream = client.readRowsCallable().call(readRowsRequest);
    for (ReadRowsResponse response : serverStream) {
        rowCount += response.getRowCount();
    }
    LOG.info(String.format("Read total of %d rows from stream '%s'.", rowCount, readStream.getName()));
    return rowCount;
}

Example 2 with ReadStream

use of com.google.cloud.bigquery.storage.v1beta2.ReadStream in project beam by apache.

the class BigQueryStorageSourceBase method split.

@Override
public List<BigQueryStorageStreamSource<T>> split(long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
    BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
    Table targetTable = getTargetTable(bqOptions);
    ReadSession.Builder readSessionBuilder = ReadSession.newBuilder();
    if (targetTable != null) {
        readSessionBuilder.setTable(BigQueryHelpers.toTableResourceName(targetTable.getTableReference()));
    } else {
        // If the table does not exist targetTable will be null.
        // Construct the table id if we can generate it. For error recording/logging.
        @Nullable String tableReferenceId = getTargetTableId(bqOptions);
        if (tableReferenceId != null) {
            readSessionBuilder.setTable(tableReferenceId);
        }
    }
    if (selectedFieldsProvider != null || rowRestrictionProvider != null) {
        ReadSession.TableReadOptions.Builder tableReadOptionsBuilder = ReadSession.TableReadOptions.newBuilder();
        if (selectedFieldsProvider != null) {
            tableReadOptionsBuilder.addAllSelectedFields(selectedFieldsProvider.get());
        }
        if (rowRestrictionProvider != null) {
            tableReadOptionsBuilder.setRowRestriction(rowRestrictionProvider.get());
        }
        readSessionBuilder.setReadOptions(tableReadOptionsBuilder);
    }
    if (format != null) {
        readSessionBuilder.setDataFormat(format);
    }
    int streamCount = 0;
    if (desiredBundleSizeBytes > 0) {
        long tableSizeBytes = (targetTable != null) ? targetTable.getNumBytes() : 0;
        streamCount = (int) Math.min(tableSizeBytes / desiredBundleSizeBytes, MAX_SPLIT_COUNT);
    }
    streamCount = Math.max(streamCount, MIN_SPLIT_COUNT);
    CreateReadSessionRequest createReadSessionRequest = CreateReadSessionRequest.newBuilder().setParent(BigQueryHelpers.toProjectResourceName(bqOptions.getBigQueryProject() == null ? bqOptions.getProject() : bqOptions.getBigQueryProject())).setReadSession(readSessionBuilder).setMaxStreamCount(streamCount).build();
    ReadSession readSession;
    try (StorageClient client = bqServices.getStorageClient(bqOptions)) {
        readSession = client.createReadSession(createReadSessionRequest);
        LOG.info("Sent BigQuery Storage API CreateReadSession request '{}'; received response '{}'.", createReadSessionRequest, readSession);
    }
    if (readSession.getStreamsList().isEmpty()) {
        // The underlying table is empty or all rows have been pruned.
        return ImmutableList.of();
    }
    Schema sessionSchema;
    if (readSession.getDataFormat() == DataFormat.ARROW) {
        org.apache.arrow.vector.types.pojo.Schema schema = ArrowConversion.arrowSchemaFromInput(readSession.getArrowSchema().getSerializedSchema().newInput());
        org.apache.beam.sdk.schemas.Schema beamSchema = ArrowConversion.ArrowSchemaTranslator.toBeamSchema(schema);
        sessionSchema = AvroUtils.toAvroSchema(beamSchema);
    } else if (readSession.getDataFormat() == DataFormat.AVRO) {
        sessionSchema = new Schema.Parser().parse(readSession.getAvroSchema().getSchema());
    } else {
        throw new IllegalArgumentException("data is not in a supported dataFormat: " + readSession.getDataFormat());
    }
    TableSchema trimmedSchema = BigQueryAvroUtils.trimBigQueryTableSchema(targetTable.getSchema(), sessionSchema);
    List<BigQueryStorageStreamSource<T>> sources = Lists.newArrayList();
    for (ReadStream readStream : readSession.getStreamsList()) {
        sources.add(BigQueryStorageStreamSource.create(readSession, readStream, trimmedSchema, parseFn, outputCoder, bqServices));
    }
    return ImmutableList.copyOf(sources);
}

Example 3 with ReadStream

use of com.google.cloud.bigquery.storage.v1beta2.ReadStream in project beam by apache.

the class BigQueryIOStorageReadTest method testStreamSourceSplitAtFractionRepeatedArrow.

@Test
public void testStreamSourceSplitAtFractionRepeatedArrow() throws Exception {
    List<ReadStream> readStreams = Lists.newArrayList(ReadStream.newBuilder().setName("stream1").build(), ReadStream.newBuilder().setName("stream2").build(), ReadStream.newBuilder().setName("stream3").build());
    StorageClient fakeStorageClient = mock(StorageClient.class);
    List<String> names = Arrays.asList("A", "B", "C", "D", "E", "F");
    List<Long> values = Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L);
    List<ReadRowsResponse> parentResponses = Lists.newArrayList(createResponseArrow(ARROW_SCHEMA, names.subList(0, 2), values.subList(0, 2), 0.0, 0.25), createResponseArrow(ARROW_SCHEMA, names.subList(2, 4), values.subList(2, 4), 0.25, 0.5), createResponseArrow(ARROW_SCHEMA, names.subList(4, 6), values.subList(4, 6), 0.5, 0.75));
    // Mock the initial ReadRows call.
    when(fakeStorageClient.readRows(ReadRowsRequest.newBuilder().setReadStream(readStreams.get(0).getName()).build(), "")).thenReturn(new FakeBigQueryServerStream<>(parentResponses));
    // Mock the first SplitReadStream call.
    when(fakeStorageClient.splitReadStream(SplitReadStreamRequest.newBuilder().setName(readStreams.get(0).getName()).setFraction(0.83f).build())).thenReturn(SplitReadStreamResponse.newBuilder().setPrimaryStream(readStreams.get(1)).setRemainderStream(ReadStream.newBuilder().setName("ignored")).build());
    List<ReadRowsResponse> otherResponses = Lists.newArrayList(createResponseArrow(ARROW_SCHEMA, names.subList(1, 3), values.subList(1, 3), 0.0, 0.50), createResponseArrow(ARROW_SCHEMA, names.subList(3, 4), values.subList(3, 4), 0.5, 0.75));
    // Mock the second ReadRows call.
    when(fakeStorageClient.readRows(ReadRowsRequest.newBuilder().setReadStream(readStreams.get(1).getName()).setOffset(1).build(), "")).thenReturn(new FakeBigQueryServerStream<>(otherResponses));
    // Mock the second SplitReadStream call.
    when(fakeStorageClient.splitReadStream(SplitReadStreamRequest.newBuilder().setName(readStreams.get(1).getName()).setFraction(0.75f).build())).thenReturn(SplitReadStreamResponse.newBuilder().setPrimaryStream(readStreams.get(2)).setRemainderStream(ReadStream.newBuilder().setName("ignored")).build());
    List<ReadRowsResponse> lastResponses = Lists.newArrayList(createResponseArrow(ARROW_SCHEMA, names.subList(2, 4), values.subList(2, 4), 0.80, 0.90));
    // Mock the third ReadRows call.
    when(fakeStorageClient.readRows(ReadRowsRequest.newBuilder().setReadStream(readStreams.get(2).getName()).setOffset(2).build(), "")).thenReturn(new FakeBigQueryServerStream<>(lastResponses));
    BoundedSource<TableRow> source = BigQueryStorageStreamSource.create(ReadSession.newBuilder().setName("readSession").setArrowSchema(ArrowSchema.newBuilder().setSerializedSchema(serializeArrowSchema(ARROW_SCHEMA)).build()).setDataFormat(DataFormat.ARROW).build(), readStreams.get(0), TABLE_SCHEMA, new TableRowParser(), TableRowJsonCoder.of(), new FakeBigQueryServices().withStorageClient(fakeStorageClient));
    BoundedReader<TableRow> reader = source.createReader(options);
    assertTrue(reader.start());
    assertEquals("A", reader.getCurrent().get("name"));
    BoundedSource<TableRow> residualSource = reader.splitAtFraction(0.83f);
    assertNotNull(residualSource);
    assertEquals("A", reader.getCurrent().get("name"));
    assertTrue(reader.advance());
    assertEquals("B", reader.getCurrent().get("name"));
    residualSource = reader.splitAtFraction(0.75f);
    assertNotNull(residualSource);
    assertEquals("B", reader.getCurrent().get("name"));
    assertTrue(reader.advance());
    assertEquals("C", reader.getCurrent().get("name"));
    assertTrue(reader.advance());
    assertEquals("D", reader.getCurrent().get("name"));
    assertFalse(reader.advance());
}

Example 4 with ReadStream

use of com.google.cloud.bigquery.storage.v1beta2.ReadStream in project beam by apache.

the class BigQueryIOStorageReadTest method testStreamSourceSplitAtFractionRepeated.

@Test
public void testStreamSourceSplitAtFractionRepeated() throws Exception {
    List<ReadStream> readStreams = Lists.newArrayList(ReadStream.newBuilder().setName("stream1").build(), ReadStream.newBuilder().setName("stream2").build(), ReadStream.newBuilder().setName("stream3").build());
    StorageClient fakeStorageClient = mock(StorageClient.class);
    // Mock the initial ReadRows call.
    when(fakeStorageClient.readRows(ReadRowsRequest.newBuilder().setReadStream(readStreams.get(0).getName()).build(), "")).thenReturn(new FakeBigQueryServerStream<>(Lists.newArrayList(createResponse(AVRO_SCHEMA, Lists.newArrayList(createRecord("A", 1, AVRO_SCHEMA), createRecord("B", 2, AVRO_SCHEMA)), 0.0, 0.25), createResponse(AVRO_SCHEMA, Lists.newArrayList(createRecord("C", 3, AVRO_SCHEMA), createRecord("D", 4, AVRO_SCHEMA)), 0.25, 0.50), createResponse(AVRO_SCHEMA, Lists.newArrayList(createRecord("E", 5, AVRO_SCHEMA), createRecord("F", 6, AVRO_SCHEMA)), 0.5, 0.75))));
    // Mock the first SplitReadStream call.
    when(fakeStorageClient.splitReadStream(SplitReadStreamRequest.newBuilder().setName(readStreams.get(0).getName()).setFraction(0.83f).build())).thenReturn(SplitReadStreamResponse.newBuilder().setPrimaryStream(readStreams.get(1)).setRemainderStream(ReadStream.newBuilder().setName("ignored")).build());
    // Mock the second ReadRows call.
    when(fakeStorageClient.readRows(ReadRowsRequest.newBuilder().setReadStream(readStreams.get(1).getName()).setOffset(1).build(), "")).thenReturn(new FakeBigQueryServerStream<>(Lists.newArrayList(createResponse(AVRO_SCHEMA, Lists.newArrayList(createRecord("B", 2, AVRO_SCHEMA), createRecord("C", 3, AVRO_SCHEMA)), 0.0, 0.50), createResponse(AVRO_SCHEMA, Lists.newArrayList(createRecord("D", 4, AVRO_SCHEMA), createRecord("E", 5, AVRO_SCHEMA)), 0.5, 0.75))));
    // Mock the second SplitReadStream call.
    when(fakeStorageClient.splitReadStream(SplitReadStreamRequest.newBuilder().setName(readStreams.get(1).getName()).setFraction(0.75f).build())).thenReturn(SplitReadStreamResponse.newBuilder().setPrimaryStream(readStreams.get(2)).setRemainderStream(ReadStream.newBuilder().setName("ignored")).build());
    // Mock the third ReadRows call.
    when(fakeStorageClient.readRows(ReadRowsRequest.newBuilder().setReadStream(readStreams.get(2).getName()).setOffset(2).build(), "")).thenReturn(new FakeBigQueryServerStream<>(Lists.newArrayList(createResponse(AVRO_SCHEMA, Lists.newArrayList(createRecord("C", 3, AVRO_SCHEMA), createRecord("D", 4, AVRO_SCHEMA)), 0.80, 0.90))));
    BoundedSource<TableRow> source = BigQueryStorageStreamSource.create(ReadSession.newBuilder().setName("readSession").setAvroSchema(AvroSchema.newBuilder().setSchema(AVRO_SCHEMA_STRING)).build(), readStreams.get(0), TABLE_SCHEMA, new TableRowParser(), TableRowJsonCoder.of(), new FakeBigQueryServices().withStorageClient(fakeStorageClient));
    BoundedReader<TableRow> reader = source.createReader(options);
    assertTrue(reader.start());
    assertEquals("A", reader.getCurrent().get("name"));
    BoundedSource<TableRow> residualSource = reader.splitAtFraction(0.83f);
    assertNotNull(residualSource);
    assertEquals("A", reader.getCurrent().get("name"));
    assertTrue(reader.advance());
    assertEquals("B", reader.getCurrent().get("name"));
    residualSource = reader.splitAtFraction(0.75f);
    assertNotNull(residualSource);
    assertEquals("B", reader.getCurrent().get("name"));
    assertTrue(reader.advance());
    assertEquals("C", reader.getCurrent().get("name"));
    assertTrue(reader.advance());
    assertEquals("D", reader.getCurrent().get("name"));
    assertFalse(reader.advance());
}

Example 5 with ReadStream

use of com.google.cloud.bigquery.storage.v1beta2.ReadStream in project spark-bigquery-connector by GoogleCloudDataproc.

the class BigQueryDataSourceReaderContext method planBatchInputPartitionContexts.

public Stream<InputPartitionContext<ColumnarBatch>> planBatchInputPartitionContexts() {
    if (!enableBatchRead()) {
        throw new IllegalStateException("Batch reads should not be enabled");
    }
    ImmutableList<String> selectedFields = schema.map(requiredSchema -> ImmutableList.copyOf(requiredSchema.fieldNames())).orElse(ImmutableList.copyOf(fields.keySet()));
    Optional<String> filter = getCombinedFilter();
    ReadSessionResponse readSessionResponse = readSessionCreator.create(tableId, selectedFields, filter);
    ReadSession readSession = readSessionResponse.getReadSession();
    logger.info("Created read session for {}: {} for application id: {}", tableId.toString(), readSession.getName(), applicationId);
    if (selectedFields.isEmpty()) {
        // means select *
        Schema tableSchema = SchemaConverters.getSchemaWithPseudoColumns(readSessionResponse.getReadTableInfo());
        selectedFields = tableSchema.getFields().stream().map(Field::getName).collect(ImmutableList.toImmutableList());
    }
    ImmutableList<String> partitionSelectedFields = selectedFields;
    return Streams.stream(Iterables.partition(readSession.getStreamsList(), readSessionCreatorConfig.streamsPerPartition())).map(streams -> new ArrowInputPartitionContext(bigQueryReadClientFactory, bigQueryTracerFactory, streams.stream().map(ReadStream::getName).collect(Collectors.toCollection(ArrayList::new)), readSessionCreatorConfig.toReadRowsHelperOptions(), partitionSelectedFields, readSessionResponse, userProvidedSchema));
}