Examples with TableRateLimiter org.apache.samza.table.remote.TableRateLimiter used on opensource projects

Example 1 with TableRateLimiter

use of org.apache.samza.table.remote.TableRateLimiter in project samza by apache.

the class TestRemoteTableDescriptor method doTestDeserializeReadFunctionAndLimiter.

private void doTestDeserializeReadFunctionAndLimiter(boolean rateOnly, boolean rlGets, boolean rlPuts) {
    int numRateLimitOps = (rlGets ? 1 : 0) + (rlPuts ? 1 : 0);
    RemoteTableDescriptor<String, String, String> desc = new RemoteTableDescriptor("1").withReadFunction(createMockTableReadFunction()).withReadRetryPolicy(new TableRetryPolicy().withRetryPredicate((ex) -> false)).withWriteFunction(createMockTableWriteFunction()).withAsyncCallbackExecutorPoolSize(10);
    if (rateOnly) {
        if (rlGets) {
            desc.withReadRateLimit(1000);
        } else {
            desc.withReadRateLimiterDisabled();
        }
        if (rlPuts) {
            desc.withWriteRateLimit(2000);
        } else {
            desc.withWriteRateLimiterDisabled();
        }
    } else {
        if (numRateLimitOps > 0) {
            Map<String, Integer> tagCredits = new HashMap<>();
            if (rlGets) {
                tagCredits.put(RemoteTableDescriptor.RL_READ_TAG, 1000);
            } else {
                desc.withReadRateLimiterDisabled();
            }
            if (rlPuts) {
                tagCredits.put(RemoteTableDescriptor.RL_WRITE_TAG, 2000);
            } else {
                desc.withWriteRateLimiterDisabled();
            }
            // Spy the rate limiter to verify call count
            RateLimiter rateLimiter = spy(new EmbeddedTaggedRateLimiter(tagCredits));
            desc.withRateLimiter(rateLimiter, new CountingCreditFunction(), new CountingCreditFunction());
        } else {
            desc.withRateLimiterDisabled();
        }
    }
    RemoteTableProvider provider = new RemoteTableProvider(desc.getTableId());
    provider.init(createMockContext(desc));
    Table table = provider.getTable();
    Assert.assertTrue(table instanceof RemoteTable);
    RemoteTable rwTable = (RemoteTable) table;
    AsyncReadWriteUpdateTable delegate = TestUtils.getFieldValue(rwTable, "asyncTable");
    Assert.assertTrue(delegate instanceof AsyncRetriableTable);
    if (rlGets || rlPuts) {
        delegate = TestUtils.getFieldValue(delegate, "table");
        Assert.assertTrue(delegate instanceof AsyncRateLimitedTable);
    }
    delegate = TestUtils.getFieldValue(delegate, "table");
    Assert.assertTrue(delegate instanceof AsyncRemoteTable);
    if (numRateLimitOps > 0) {
        TableRateLimiter readRateLimiter = TestUtils.getFieldValue(rwTable, "readRateLimiter");
        TableRateLimiter writeRateLimiter = TestUtils.getFieldValue(rwTable, "writeRateLimiter");
        Assert.assertTrue(!rlGets || readRateLimiter != null);
        Assert.assertTrue(!rlPuts || writeRateLimiter != null);
    }
    ThreadPoolExecutor callbackExecutor = TestUtils.getFieldValue(rwTable, "callbackExecutor");
    Assert.assertEquals(10, callbackExecutor.getCorePoolSize());
}

Example 2 with TableRateLimiter

use of org.apache.samza.table.remote.TableRateLimiter in project samza by apache.

the class TestAsyncRateLimitedTable method testFlushAndClose.

@Test
public void testFlushAndClose() {
    TableRateLimiter readRateLimiter = mock(TableRateLimiter.class);
    TableRateLimiter writeRateLimiter = mock(TableRateLimiter.class);
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writeFn = mock(TableWriteFunction.class);
    AsyncReadWriteUpdateTable delegate = new AsyncRemoteTable(readFn, writeFn);
    AsyncRateLimitedTable table = new AsyncRateLimitedTable("t1", delegate, readRateLimiter, writeRateLimiter, writeRateLimiter, schedExec);
    table.init(TestRemoteTable.getMockContext());
    table.flush();
    verify(writeFn, times(1)).flush();
    table.close();
    verify(readFn, times(1)).close();
    verify(writeFn, times(1)).close();
}

Example 3 with TableRateLimiter

use of org.apache.samza.table.remote.TableRateLimiter in project samza by apache.

the class TestRemoteTableEndToEnd method testUninitializedWriter.

@Test
public void testUninitializedWriter() {
    TableReadFunction<String, String> reader = mock(TableReadFunction.class);
    TableRateLimiter rateLimitHelper = mock(TableRateLimiter.class);
    RemoteTable<String, String, Void> table = new RemoteTable<>("table1", reader, null, rateLimitHelper, null, null, Executors.newSingleThreadExecutor(), null, null, null, null, null, null);
    table.init(createMockContext());
    try {
        table.put("abc", "efg");
        Assert.fail();
    } catch (SamzaException ex) {
    // Ignore
    }
    try {
        table.delete("abc");
        Assert.fail();
    } catch (SamzaException ex) {
    // Ignore
    }
    table.flush();
    table.close();
}

Example 4 with TableRateLimiter

use of org.apache.samza.table.remote.TableRateLimiter in project samza by apache.

the class TestRemoteTableEndToEnd method testCatchWriterExceptionWithUpdates.

@Test(expected = SamzaException.class)
public void testCatchWriterExceptionWithUpdates() {
    TableReadFunction<String, String> reader = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writer = mock(TableWriteFunction.class);
    CompletableFuture<String> future = new CompletableFuture<>();
    future.completeExceptionally(new RuntimeException("Expected test exception"));
    doReturn(future).when(writer).updateAsync(anyString(), anyString());
    TableRateLimiter rateLimitHelper = mock(TableRateLimiter.class);
    RemoteTable<String, String, String> table = new RemoteTable<>("table1", reader, writer, rateLimitHelper, rateLimitHelper, rateLimitHelper, Executors.newSingleThreadExecutor(), null, null, null, null, null, null);
    table.init(createMockContext());
    table.update("abc", "xyz");
}

Example 5 with TableRateLimiter

use of org.apache.samza.table.remote.TableRateLimiter in project samza by apache.

the class TestRemoteTableEndToEnd method testCatchReaderException.

@Test(expected = SamzaException.class)
public void testCatchReaderException() {
    TableReadFunction<String, ?> reader = mock(TableReadFunction.class);
    CompletableFuture<String> future = new CompletableFuture<>();
    future.completeExceptionally(new RuntimeException("Expected test exception"));
    doReturn(future).when(reader).getAsync(anyString());
    TableRateLimiter rateLimitHelper = mock(TableRateLimiter.class);
    RemoteTable<String, String, Void> table = new RemoteTable<>("table1", reader, null, rateLimitHelper, null, null, Executors.newSingleThreadExecutor(), null, null, null, null, null, null);
    table.init(createMockContext());
    table.get("abc");
}