Examples with ReadRowsResponse com.google.bigtable.v2.ReadRowsResponse used on opensource projects

Example 26 with ReadRowsResponse

use of com.google.bigtable.v2.ReadRowsResponse in project beam by apache.

the class BigQueryIOStorageQueryTest method doReadFromBigQueryIO.

private void doReadFromBigQueryIO(boolean templateCompatibility) throws Exception {
    TableReference sourceTableRef = BigQueryHelpers.parseTableSpec("project:dataset.table");
    fakeDatasetService.createDataset(sourceTableRef.getProjectId(), sourceTableRef.getDatasetId(), "asia-northeast1", "Fake plastic tree^H^H^H^Htables", null);
    fakeDatasetService.createTable(new Table().setTableReference(sourceTableRef).setLocation("asia-northeast1"));
    Table queryResultTable = new Table().setSchema(TABLE_SCHEMA).setNumBytes(0L);
    String encodedQuery = FakeBigQueryServices.encodeQueryResult(queryResultTable);
    fakeJobService.expectDryRunQuery(options.getProject(), encodedQuery, new JobStatistics().setQuery(new JobStatistics2().setTotalBytesProcessed(1024L * 1024L).setReferencedTables(ImmutableList.of(sourceTableRef))));
    ReadSession readSession = ReadSession.newBuilder().setName("readSessionName").setAvroSchema(AvroSchema.newBuilder().setSchema(AVRO_SCHEMA_STRING)).addStreams(ReadStream.newBuilder().setName("streamName")).setDataFormat(DataFormat.AVRO).build();
    ReadRowsRequest expectedReadRowsRequest = ReadRowsRequest.newBuilder().setReadStream("streamName").build();
    List<GenericRecord> records = Lists.newArrayList(createRecord("A", 1, AVRO_SCHEMA), createRecord("B", 2, AVRO_SCHEMA), createRecord("C", 3, AVRO_SCHEMA), createRecord("D", 4, AVRO_SCHEMA));
    List<ReadRowsResponse> readRowsResponses = Lists.newArrayList(createResponse(AVRO_SCHEMA, records.subList(0, 2), 0.0, 0.500), createResponse(AVRO_SCHEMA, records.subList(2, 4), 0.5, 0.875));
    // 
    // Note that since the temporary table name is generated by the pipeline, we can't match the
    // expected create read session request exactly. For now, match against any appropriately typed
    // proto object.
    // 
    StorageClient fakeStorageClient = mock(StorageClient.class, withSettings().serializable());
    when(fakeStorageClient.createReadSession(any())).thenReturn(readSession);
    when(fakeStorageClient.readRows(expectedReadRowsRequest, "")).thenReturn(new FakeBigQueryServerStream<>(readRowsResponses));
    BigQueryIO.TypedRead<KV<String, Long>> typedRead = BigQueryIO.read(new ParseKeyValue()).fromQuery(encodedQuery).withMethod(Method.DIRECT_READ).withTestServices(new FakeBigQueryServices().withDatasetService(fakeDatasetService).withJobService(fakeJobService).withStorageClient(fakeStorageClient));
    if (templateCompatibility) {
        typedRead = typedRead.withTemplateCompatibility();
    }
    PCollection<KV<String, Long>> output = p.apply(typedRead);
    PAssert.that(output).containsInAnyOrder(ImmutableList.of(KV.of("A", 1L), KV.of("B", 2L), KV.of("C", 3L), KV.of("D", 4L)));
    p.run();
}

Example 27 with ReadRowsResponse

use of com.google.bigtable.v2.ReadRowsResponse in project java-bigtable by googleapis.

the class MetricsTracerTest method testBatchReadRowsThrottledTime.

@Test
public void testBatchReadRowsThrottledTime() throws Exception {
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) {
            @SuppressWarnings("unchecked") StreamObserver<ReadRowsResponse> observer = (StreamObserver<ReadRowsResponse>) invocation.getArguments()[1];
            observer.onNext(DEFAULT_READ_ROWS_RESPONSES);
            observer.onCompleted();
            return null;
        }
    }).when(mockService).readRows(any(ReadRowsRequest.class), any());
    try (Batcher batcher = stub.newBulkReadRowsBatcher(Query.create(TABLE_ID), GrpcCallContext.createDefault())) {
        batcher.add(ByteString.copyFromUtf8("row1"));
        batcher.sendOutstanding();
        // Give OpenCensus a chance to update the views asynchronously.
        Thread.sleep(100);
        long throttledTimeMetric = StatsTestUtils.getAggregationValueAsLong(localStats, RpcViewConstants.BIGTABLE_BATCH_THROTTLED_TIME_VIEW, ImmutableMap.of(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows")), PROJECT_ID, INSTANCE_ID, APP_PROFILE_ID);
        assertThat(throttledTimeMetric).isEqualTo(0);
    }
}

Example 28 with ReadRowsResponse

use of com.google.bigtable.v2.ReadRowsResponse in project java-bigtable by googleapis.

the class MetricsTracerTest method testReadRowsAttemptLatency.

@Test
public void testReadRowsAttemptLatency() throws InterruptedException {
    final long sleepTime = 50;
    final AtomicInteger callCount = new AtomicInteger(0);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            @SuppressWarnings("unchecked") StreamObserver<ReadRowsResponse> observer = (StreamObserver<ReadRowsResponse>) invocation.getArguments()[1];
            Thread.sleep(sleepTime);
            // First attempt will return a transient error
            if (callCount.getAndIncrement() == 0) {
                observer.onError(new StatusRuntimeException(Status.UNAVAILABLE));
                return null;
            }
            // Next attempt will be ok
            observer.onNext(DEFAULT_READ_ROWS_RESPONSES);
            observer.onCompleted();
            return null;
        }
    }).when(mockService).readRows(any(ReadRowsRequest.class), any());
    Stopwatch stopwatch = Stopwatch.createStarted();
    Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID)));
    long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
    // Give OpenCensus a chance to update the views asynchronously.
    Thread.sleep(100);
    long attemptLatency = StatsTestUtils.getAggregationValueAsLong(localStats, RpcViewConstants.BIGTABLE_ATTEMPT_LATENCY_VIEW, ImmutableMap.of(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows"), RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK")), PROJECT_ID, INSTANCE_ID, APP_PROFILE_ID);
    // Average attempt latency will be just a single wait (as opposed to op latency which will be 2x
    // sleeptime)
    assertThat(attemptLatency).isIn(Range.closed(sleepTime, elapsed - sleepTime));
}

Example 29 with ReadRowsResponse

use of com.google.bigtable.v2.ReadRowsResponse in project java-bigtable by googleapis.

the class MetricsTracerTest method testReadRowsOpCount.

@Test
public void testReadRowsOpCount() throws InterruptedException {
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) {
            @SuppressWarnings("unchecked") StreamObserver<ReadRowsResponse> observer = (StreamObserver<ReadRowsResponse>) invocation.getArguments()[1];
            observer.onNext(DEFAULT_READ_ROWS_RESPONSES);
            observer.onCompleted();
            return null;
        }
    }).when(mockService).readRows(any(ReadRowsRequest.class), any());
    Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID)));
    Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID)));
    // Give OpenCensus a chance to update the views asynchronously.
    Thread.sleep(100);
    long opLatency = StatsTestUtils.getAggregationValueAsLong(localStats, RpcViewConstants.BIGTABLE_COMPLETED_OP_VIEW, ImmutableMap.of(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows"), RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK")), PROJECT_ID, INSTANCE_ID, APP_PROFILE_ID);
    assertThat(opLatency).isEqualTo(2);
}

Example 30 with ReadRowsResponse

use of com.google.bigtable.v2.ReadRowsResponse in project java-bigtable by googleapis.

the class SimpleGceRawRead method main.

public static void main(String[] args) {
    configureLogging();
    String projectId = System.getProperty("bigtable.project", "gcloud-devel");
    String instanceId = System.getProperty("bigtable.instance", "google-cloud-bigtable");
    String tableId = System.getProperty("bigtable.table", "integration-tests");
    String endpoint = System.getProperty("bigtable.data-endpoint", "bigtable.googleapis.com");
    String tableName = String.format("projects/%s/instances/%s/tables/%s", projectId, instanceId, tableId);
    System.out.printf(">>>>>>>>> Trying to connect to: %s, to read %s%n%n%n", endpoint, tableName);
    ManagedChannel channel = ComputeEngineChannelBuilder.forAddress(endpoint, 443).disableServiceConfigLookUp().defaultServiceConfig(newServiceConfig()).build();
    try {
        BigtableGrpc.BigtableBlockingStub stub = BigtableGrpc.newBlockingStub(channel);
        Iterator<ReadRowsResponse> iter = stub.readRows(ReadRowsRequest.newBuilder().setTableName(tableName).setRowsLimit(1).build());
        System.out.printf("%n%n>>>>>>>>> Success Rows Read: %d%n%n", Lists.newArrayList(iter).size());
    } finally {
        channel.shutdown();
    }
}