Examples with BatcherImpl com.google.api.gax.batching.BatcherImpl used on opensource projects

Example 1 with BatcherImpl

use of com.google.api.gax.batching.BatcherImpl in project java-bigtable by googleapis.

the class EnhancedBigtableStubTest method testBulkMutationFlowControllerConfigured.

@Test
public void testBulkMutationFlowControllerConfigured() throws Exception {
    BigtableDataSettings.Builder settings = BigtableDataSettings.newBuilder().setProjectId("my-project").setInstanceId("my-instance").setCredentialsProvider(defaultSettings.getCredentialsProvider()).enableBatchMutationLatencyBasedThrottling(10L);
    settings.stubSettings().bulkMutateRowsSettings().setBatchingSettings(BatchingSettings.newBuilder().setElementCountThreshold(50L).setRequestByteThreshold(500L).setFlowControlSettings(FlowControlSettings.newBuilder().setMaxOutstandingElementCount(100L).setMaxOutstandingRequestBytes(1000L).setLimitExceededBehavior(LimitExceededBehavior.Block).build()).build()).build();
    try (EnhancedBigtableStub stub1 = EnhancedBigtableStub.create(settings.build().getStubSettings());
        EnhancedBigtableStub stub2 = EnhancedBigtableStub.create(settings.build().getStubSettings())) {
        // FlowControlEventStats
        try (BatcherImpl batcher1 = (BatcherImpl) stub1.newMutateRowsBatcher("my-table1", null);
            BatcherImpl batcher2 = (BatcherImpl) stub1.newMutateRowsBatcher("my-table2", null)) {
            assertThat(batcher1.getFlowController()).isNotNull();
            assertThat(batcher1.getFlowController().getFlowControlEventStats()).isNotNull();
            assertThat(batcher1).isNotSameInstanceAs(batcher2);
            assertThat(batcher1.getFlowController()).isSameInstanceAs(batcher2.getFlowController());
            assertThat(batcher1.getFlowController().getFlowControlEventStats()).isSameInstanceAs(batcher2.getFlowController().getFlowControlEventStats());
            // Verify flow controller settings
            assertThat(batcher1.getFlowController().getMaxElementCountLimit()).isEqualTo(100L);
            assertThat(batcher1.getFlowController().getMaxRequestBytesLimit()).isEqualTo(1000L);
            assertThat(batcher1.getFlowController().getCurrentElementCountLimit()).isLessThan(100L);
            assertThat(batcher1.getFlowController().getCurrentRequestBytesLimit()).isEqualTo(1000L);
            assertThat(batcher1.getFlowController().getMinElementCountLimit()).isAtLeast(settings.stubSettings().bulkMutateRowsSettings().getBatchingSettings().getElementCountThreshold());
            assertThat(batcher1.getFlowController().getMinRequestBytesLimit()).isEqualTo(1000L);
        }
        // FlowControlEventStats
        try (BatcherImpl batcher1 = (BatcherImpl) stub1.newMutateRowsBatcher("my-table1", null);
            BatcherImpl batcher2 = (BatcherImpl) stub2.newMutateRowsBatcher("my-table2", null)) {
            assertThat(batcher1.getFlowController()).isNotNull();
            assertThat(batcher1.getFlowController().getFlowControlEventStats()).isNotNull();
            assertThat(batcher1.getFlowController()).isNotSameInstanceAs(batcher2.getFlowController());
            assertThat(batcher1.getFlowController().getFlowControlEventStats()).isNotSameInstanceAs(batcher2.getFlowController().getFlowControlEventStats());
        }
    }
    try (EnhancedBigtableStub stub1 = EnhancedBigtableStub.create(settings.build().getStubSettings());
        EnhancedBigtableStub stub2 = EnhancedBigtableStub.create(settings.disableBatchMutationLatencyBasedThrottling().build().getStubSettings())) {
        try (BatcherImpl batcher = (BatcherImpl) stub2.newMutateRowsBatcher("my-table", null)) {
            assertThat(batcher.getFlowController().getMaxElementCountLimit()).isEqualTo(100L);
            assertThat(batcher.getFlowController().getCurrentElementCountLimit()).isEqualTo(100L);
            assertThat(batcher.getFlowController().getMinElementCountLimit()).isEqualTo(100L);
        }
    }
}

Example 2 with BatcherImpl

use of com.google.api.gax.batching.BatcherImpl in project java-bigtable by googleapis.

the class MetricsTracerTest method testBatchMutateRowsThrottledTime.

@Test
public void testBatchMutateRowsThrottledTime() throws Exception {
    FlowController flowController = Mockito.mock(FlowController.class);
    BatchingDescriptor batchingDescriptor = Mockito.mock(MutateRowsBatchingDescriptor.class);
    // Mock throttling
    final long throttled = 50;
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Thread.sleep(throttled);
            return null;
        }
    }).when(flowController).reserve(any(Long.class), any(Long.class));
    when(flowController.getMaxElementCountLimit()).thenReturn(null);
    when(flowController.getMaxRequestBytesLimit()).thenReturn(null);
    when(batchingDescriptor.countBytes(any())).thenReturn(1l);
    when(batchingDescriptor.newRequestBuilder(any())).thenCallRealMethod();
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) {
            @SuppressWarnings("unchecked") StreamObserver<MutateRowsResponse> observer = (StreamObserver<MutateRowsResponse>) invocation.getArguments()[1];
            observer.onNext(MutateRowsResponse.getDefaultInstance());
            observer.onCompleted();
            return null;
        }
    }).when(mockService).mutateRows(any(MutateRowsRequest.class), any());
    ApiCallContext defaultContext = GrpcCallContext.createDefault();
    Batcher batcher = new BatcherImpl(batchingDescriptor, stub.bulkMutateRowsCallable().withDefaultCallContext(defaultContext), BulkMutation.create(TABLE_ID), settings.getStubSettings().bulkMutateRowsSettings().getBatchingSettings(), Executors.newSingleThreadScheduledExecutor(), flowController, defaultContext);
    batcher.add(RowMutationEntry.create("key"));
    batcher.sendOutstanding();
    Thread.sleep(100);
    long throttledTimeMetric = StatsTestUtils.getAggregationValueAsLong(localStats, RpcViewConstants.BIGTABLE_BATCH_THROTTLED_TIME_VIEW, ImmutableMap.of(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.MutateRows")), PROJECT_ID, INSTANCE_ID, APP_PROFILE_ID);
    assertThat(throttledTimeMetric).isAtLeast(throttled);
}

Example 3 with BatcherImpl

use of com.google.api.gax.batching.BatcherImpl in project java-bigtable-hbase by googleapis.

the class TestBulkMutationVeneerApi method testWhenBatcherIsClosed.

@Test
public void testWhenBatcherIsClosed() throws IOException {
    BatchingSettings batchingSettings = mock(BatchingSettings.class);
    FlowControlSettings flowControlSettings = FlowControlSettings.newBuilder().setLimitExceededBehavior(LimitExceededBehavior.Ignore).build();
    when(batchingSettings.getFlowControlSettings()).thenReturn(flowControlSettings);
    @SuppressWarnings("unchecked") Batcher<RowMutationEntry, Void> actualBatcher = new BatcherImpl(mock(BatchingDescriptor.class), mock(UnaryCallable.class), new Object(), batchingSettings, mock(ScheduledExecutorService.class));
    BulkMutationWrapper underTest = new BulkMutationVeneerApi(actualBatcher);
    underTest.close();
    Exception actualEx = null;
    try {
        underTest.add(rowMutation);
        fail("batcher should throw exception");
    } catch (Exception e) {
        actualEx = e;
    }
    assertTrue(actualEx instanceof IllegalStateException);
}

Example 4 with BatcherImpl

use of com.google.api.gax.batching.BatcherImpl in project java-bigtable by googleapis.

the class BigtableDataClientFactoryTest method testBulkMutationFlowControllerConfigured.

@Test
public void testBulkMutationFlowControllerConfigured() throws Exception {
    BigtableDataSettings settings = BigtableDataSettings.newBuilder().setProjectId("my-project").setInstanceId("my-instance").setCredentialsProvider(credentialsProvider).enableBatchMutationLatencyBasedThrottling(10L).build();
    try (BigtableDataClientFactory factory = BigtableDataClientFactory.create(settings)) {
        BigtableDataClient client1 = factory.createDefault();
        BigtableDataClient client2 = factory.createForAppProfile("app-profile");
        try (BatcherImpl batcher1 = (BatcherImpl) client1.newBulkMutationBatcher("my-table");
            BatcherImpl batcher2 = (BatcherImpl) client1.newBulkMutationBatcher("my-table")) {
            assertThat(batcher1.getFlowController()).isSameInstanceAs(batcher2.getFlowController());
        }
        try (BatcherImpl batcher1 = (BatcherImpl) client1.newBulkMutationBatcher("my-table");
            BatcherImpl batcher2 = (BatcherImpl) client2.newBulkMutationBatcher("my-table")) {
            assertThat(batcher1.getFlowController()).isNotSameInstanceAs(batcher2.getFlowController());
        }
    }
}

Example 5 with BatcherImpl

use of com.google.api.gax.batching.BatcherImpl in project java-bigtable by googleapis.

the class BulkMutateIT method test.

@Test(timeout = 60 * 1000)
public void test() throws IOException, InterruptedException {
    BigtableDataSettings settings = testEnvRule.env().getDataClientSettings();
    String rowPrefix = UUID.randomUUID().toString();
    // Set target latency really low so it'll trigger adjusting thresholds
    BigtableDataSettings.Builder builder = settings.toBuilder().enableBatchMutationLatencyBasedThrottling(2L);
    try (BigtableDataClient client = BigtableDataClient.create(builder.build());
        BatcherImpl<RowMutationEntry, Void, BulkMutation, Void> batcher = (BatcherImpl<RowMutationEntry, Void, BulkMutation, Void>) client.newBulkMutationBatcher(testEnvRule.env().getTableId())) {
        FlowControlEventStats events = batcher.getFlowController().getFlowControlEventStats();
        long initialThreashold = Objects.requireNonNull(batcher.getFlowController().getCurrentElementCountLimit());
        assertThat(batcher.getFlowController().getCurrentElementCountLimit()).isNotEqualTo(batcher.getFlowController().getMinElementCountLimit());
        assertThat(batcher.getFlowController().getCurrentElementCountLimit()).isNotEqualTo(batcher.getFlowController().getMaxElementCountLimit());
        String familyId = testEnvRule.env().getFamilyId();
        long initial = batcher.getFlowController().getCurrentElementCountLimit();
        for (long i = 0; i < initial * 3; i++) {
            String key = rowPrefix + "test-key" + i;
            batcher.add(RowMutationEntry.create(key).setCell(familyId, "qualifier", i));
        }
        batcher.flush();
        assertThat(events.getLastFlowControlEvent()).isNotNull();
        // Verify that the threshold is adjusted
        assertThat(batcher.getFlowController().getCurrentElementCountLimit()).isNotEqualTo(initialThreashold);
        // Query a key to make sure the write succeeded
        Row row = testEnvRule.env().getDataClient().readRowsCallable().first().call(Query.create(testEnvRule.env().getTableId()).rowKey(rowPrefix + "test-key" + initial));
        assertThat(row.getCells()).hasSize(1);
    }
}