Examples with AsyncRemoteTable org.apache.samza.table.remote.AsyncRemoteTable used on opensource projects

Example 1 with AsyncRemoteTable

use of org.apache.samza.table.remote.AsyncRemoteTable in project samza by apache.

the class TestRemoteTableDescriptor method doTestDeserializeReadFunctionAndLimiter.

private void doTestDeserializeReadFunctionAndLimiter(boolean rateOnly, boolean rlGets, boolean rlPuts) {
    int numRateLimitOps = (rlGets ? 1 : 0) + (rlPuts ? 1 : 0);
    RemoteTableDescriptor<String, String, String> desc = new RemoteTableDescriptor("1").withReadFunction(createMockTableReadFunction()).withReadRetryPolicy(new TableRetryPolicy().withRetryPredicate((ex) -> false)).withWriteFunction(createMockTableWriteFunction()).withAsyncCallbackExecutorPoolSize(10);
    if (rateOnly) {
        if (rlGets) {
            desc.withReadRateLimit(1000);
        } else {
            desc.withReadRateLimiterDisabled();
        }
        if (rlPuts) {
            desc.withWriteRateLimit(2000);
        } else {
            desc.withWriteRateLimiterDisabled();
        }
    } else {
        if (numRateLimitOps > 0) {
            Map<String, Integer> tagCredits = new HashMap<>();
            if (rlGets) {
                tagCredits.put(RemoteTableDescriptor.RL_READ_TAG, 1000);
            } else {
                desc.withReadRateLimiterDisabled();
            }
            if (rlPuts) {
                tagCredits.put(RemoteTableDescriptor.RL_WRITE_TAG, 2000);
            } else {
                desc.withWriteRateLimiterDisabled();
            }
            // Spy the rate limiter to verify call count
            RateLimiter rateLimiter = spy(new EmbeddedTaggedRateLimiter(tagCredits));
            desc.withRateLimiter(rateLimiter, new CountingCreditFunction(), new CountingCreditFunction());
        } else {
            desc.withRateLimiterDisabled();
        }
    }
    RemoteTableProvider provider = new RemoteTableProvider(desc.getTableId());
    provider.init(createMockContext(desc));
    Table table = provider.getTable();
    Assert.assertTrue(table instanceof RemoteTable);
    RemoteTable rwTable = (RemoteTable) table;
    AsyncReadWriteUpdateTable delegate = TestUtils.getFieldValue(rwTable, "asyncTable");
    Assert.assertTrue(delegate instanceof AsyncRetriableTable);
    if (rlGets || rlPuts) {
        delegate = TestUtils.getFieldValue(delegate, "table");
        Assert.assertTrue(delegate instanceof AsyncRateLimitedTable);
    }
    delegate = TestUtils.getFieldValue(delegate, "table");
    Assert.assertTrue(delegate instanceof AsyncRemoteTable);
    if (numRateLimitOps > 0) {
        TableRateLimiter readRateLimiter = TestUtils.getFieldValue(rwTable, "readRateLimiter");
        TableRateLimiter writeRateLimiter = TestUtils.getFieldValue(rwTable, "writeRateLimiter");
        Assert.assertTrue(!rlGets || readRateLimiter != null);
        Assert.assertTrue(!rlPuts || writeRateLimiter != null);
    }
    ThreadPoolExecutor callbackExecutor = TestUtils.getFieldValue(rwTable, "callbackExecutor");
    Assert.assertEquals(10, callbackExecutor.getCorePoolSize());
}

Example 2 with AsyncRemoteTable

use of org.apache.samza.table.remote.AsyncRemoteTable in project samza by apache.

the class TestAsyncRetriableTable method testGetWithPermFailureOnMaxCount.

@Test
public void testGetWithPermFailureOnMaxCount() {
    TableRetryPolicy policy = new TableRetryPolicy();
    policy.withFixedBackoff(Duration.ofMillis(5));
    policy.withStopAfterAttempts(10);
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    doReturn(true).when(readFn).isRetriable(any());
    CompletableFuture<String> future = new CompletableFuture();
    future.completeExceptionally(new RuntimeException("test exception"));
    doReturn(future).when(readFn).getAllAsync(any());
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, null);
    AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null);
    table.init(TestRemoteTable.getMockContext());
    try {
        table.getAsync("foo").join();
        fail();
    } catch (Throwable t) {
    }
    verify(readFn, atLeast(11)).getAsync(any());
    assertEquals(10, table.readRetryMetrics.retryCount.getCount());
    assertEquals(0, table.readRetryMetrics.successCount.getCount());
    assertEquals(1, table.readRetryMetrics.permFailureCount.getCount());
    assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0);
}

Example 3 with AsyncRemoteTable

use of org.apache.samza.table.remote.AsyncRemoteTable in project samza by apache.

the class TestAsyncRetriableTable method testGetWithoutRetry.

@Test
public void testGetWithoutRetry() {
    TableRetryPolicy policy = new TableRetryPolicy();
    policy.withFixedBackoff(Duration.ofMillis(100));
    TableReadFunction readFn = mock(TableReadFunction.class);
    doReturn(true).when(readFn).isRetriable(any());
    doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any());
    Map<String, String> result = new HashMap<>();
    result.put("foo", "bar");
    doReturn(CompletableFuture.completedFuture(result)).when(readFn).getAllAsync(any());
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, null);
    AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null);
    int times = 0;
    table.init(TestRemoteTable.getMockContext());
    verify(readFn, times(0)).init(any(), any());
    assertEquals("bar", table.getAsync("foo").join());
    verify(readFn, times(1)).getAsync(any());
    assertEquals(++times, table.readRetryMetrics.successCount.getCount());
    assertEquals(result, table.getAllAsync(Arrays.asList("foo")).join());
    verify(readFn, times(1)).getAllAsync(any());
    assertEquals(++times, table.readRetryMetrics.successCount.getCount());
    assertEquals(0, table.readRetryMetrics.retryCount.getCount());
    assertEquals(0, table.readRetryMetrics.retryTimer.getSnapshot().getMax());
    assertEquals(0, table.readRetryMetrics.permFailureCount.getCount());
    assertNull(table.writeRetryMetrics);
    table.close();
    verify(readFn, times(1)).close();
}

Example 4 with AsyncRemoteTable

use of org.apache.samza.table.remote.AsyncRemoteTable in project samza by apache.

the class TestAsyncRetriableTable method testUpdateAllWithOneRetry.

@Test
public void testUpdateAllWithOneRetry() {
    TableRetryPolicy policy = new TableRetryPolicy();
    policy.withFixedBackoff(Duration.ofMillis(10));
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writeFn = mock(TableWriteFunction.class);
    doReturn(true).when(writeFn).isRetriable(any());
    AtomicInteger times = new AtomicInteger();
    doAnswer(invocation -> {
        CompletableFuture<Map<String, String>> future = new CompletableFuture();
        if (times.get() > 0) {
            future.complete(null);
        } else {
            times.incrementAndGet();
            future.completeExceptionally(new RuntimeException("test exception"));
        }
        return future;
    }).when(writeFn).updateAllAsync(any());
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, writeFn);
    AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn);
    table.init(TestRemoteTable.getMockContext());
    table.updateAllAsync(Arrays.asList(new Entry(1, 2))).join();
    verify(writeFn, times(2)).updateAllAsync(any());
    assertEquals(1, table.writeRetryMetrics.retryCount.getCount());
    assertEquals(0, table.writeRetryMetrics.successCount.getCount());
    assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount());
    assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0);
}

Example 5 with AsyncRemoteTable

use of org.apache.samza.table.remote.AsyncRemoteTable in project samza by apache.

the class TestAsyncRetriableTable method testPutAllWithOneRetry.

@Test
public void testPutAllWithOneRetry() {
    TableRetryPolicy policy = new TableRetryPolicy();
    policy.withFixedBackoff(Duration.ofMillis(10));
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writeFn = mock(TableWriteFunction.class);
    doReturn(true).when(writeFn).isRetriable(any());
    AtomicInteger times = new AtomicInteger();
    doAnswer(invocation -> {
        CompletableFuture<Map<String, String>> future = new CompletableFuture();
        if (times.get() > 0) {
            future.complete(null);
        } else {
            times.incrementAndGet();
            future.completeExceptionally(new RuntimeException("test exception"));
        }
        return future;
    }).when(writeFn).putAllAsync(any());
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, writeFn);
    AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn);
    table.init(TestRemoteTable.getMockContext());
    table.putAllAsync(Arrays.asList(new Entry(1, 2))).join();
    verify(writeFn, times(2)).putAllAsync(any());
    assertEquals(1, table.writeRetryMetrics.retryCount.getCount());
    assertEquals(0, table.writeRetryMetrics.successCount.getCount());
    assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount());
    assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0);
}