Examples with TableRetryPolicy org.apache.samza.table.retry.TableRetryPolicy used on opensource projects

Example 1 with TableRetryPolicy

use of org.apache.samza.table.retry.TableRetryPolicy in project samza by apache.

the class TestRemoteTableDescriptor method testReadWriteRateLimitToConfig.

@Test
public void testReadWriteRateLimitToConfig() {
    Map<String, String> tableConfig = new RemoteTableDescriptor("1").withReadFunction(createMockTableReadFunction()).withReadRetryPolicy(new TableRetryPolicy()).withWriteRateLimit(1000).withReadRateLimit(2000).toConfig(new MapConfig());
    Assert.assertEquals(String.valueOf(2000), tableConfig.get("tables.1.io.read.credits"));
    Assert.assertEquals(String.valueOf(1000), tableConfig.get("tables.1.io.write.credits"));
}

Example 2 with TableRetryPolicy

use of org.apache.samza.table.retry.TableRetryPolicy in project samza by apache.

the class TestRemoteTableDescriptor method doTestDeserializeReadFunctionAndLimiter.

private void doTestDeserializeReadFunctionAndLimiter(boolean rateOnly, boolean rlGets, boolean rlPuts) {
    int numRateLimitOps = (rlGets ? 1 : 0) + (rlPuts ? 1 : 0);
    RemoteTableDescriptor<String, String, String> desc = new RemoteTableDescriptor("1").withReadFunction(createMockTableReadFunction()).withReadRetryPolicy(new TableRetryPolicy().withRetryPredicate((ex) -> false)).withWriteFunction(createMockTableWriteFunction()).withAsyncCallbackExecutorPoolSize(10);
    if (rateOnly) {
        if (rlGets) {
            desc.withReadRateLimit(1000);
        } else {
            desc.withReadRateLimiterDisabled();
        }
        if (rlPuts) {
            desc.withWriteRateLimit(2000);
        } else {
            desc.withWriteRateLimiterDisabled();
        }
    } else {
        if (numRateLimitOps > 0) {
            Map<String, Integer> tagCredits = new HashMap<>();
            if (rlGets) {
                tagCredits.put(RemoteTableDescriptor.RL_READ_TAG, 1000);
            } else {
                desc.withReadRateLimiterDisabled();
            }
            if (rlPuts) {
                tagCredits.put(RemoteTableDescriptor.RL_WRITE_TAG, 2000);
            } else {
                desc.withWriteRateLimiterDisabled();
            }
            // Spy the rate limiter to verify call count
            RateLimiter rateLimiter = spy(new EmbeddedTaggedRateLimiter(tagCredits));
            desc.withRateLimiter(rateLimiter, new CountingCreditFunction(), new CountingCreditFunction());
        } else {
            desc.withRateLimiterDisabled();
        }
    }
    RemoteTableProvider provider = new RemoteTableProvider(desc.getTableId());
    provider.init(createMockContext(desc));
    Table table = provider.getTable();
    Assert.assertTrue(table instanceof RemoteTable);
    RemoteTable rwTable = (RemoteTable) table;
    AsyncReadWriteUpdateTable delegate = TestUtils.getFieldValue(rwTable, "asyncTable");
    Assert.assertTrue(delegate instanceof AsyncRetriableTable);
    if (rlGets || rlPuts) {
        delegate = TestUtils.getFieldValue(delegate, "table");
        Assert.assertTrue(delegate instanceof AsyncRateLimitedTable);
    }
    delegate = TestUtils.getFieldValue(delegate, "table");
    Assert.assertTrue(delegate instanceof AsyncRemoteTable);
    if (numRateLimitOps > 0) {
        TableRateLimiter readRateLimiter = TestUtils.getFieldValue(rwTable, "readRateLimiter");
        TableRateLimiter writeRateLimiter = TestUtils.getFieldValue(rwTable, "writeRateLimiter");
        Assert.assertTrue(!rlGets || readRateLimiter != null);
        Assert.assertTrue(!rlPuts || writeRateLimiter != null);
    }
    ThreadPoolExecutor callbackExecutor = TestUtils.getFieldValue(rwTable, "callbackExecutor");
    Assert.assertEquals(10, callbackExecutor.getCorePoolSize());
}

Example 3 with TableRetryPolicy

use of org.apache.samza.table.retry.TableRetryPolicy in project samza by apache.

the class TestRemoteTableDescriptor method testTablePartToConfig.

@Test
public void testTablePartToConfig() {
    int key = 0;
    TableReadFunction readFn = createMockTableReadFunction();
    when(readFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableWriteFunction writeFn = createMockTableWriteFunction();
    when(writeFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    RateLimiter rateLimiter = createMockRateLimiter();
    when(((TablePart) rateLimiter).toConfig(any(), any())).thenReturn(createConfigPair(key));
    CreditFunction readCredFn = createMockCreditFunction();
    when(readCredFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    CreditFunction writeCredFn = createMockCreditFunction();
    when(writeCredFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableRetryPolicy readRetryPolicy = createMockTableRetryPolicy();
    when(readRetryPolicy.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableRetryPolicy writeRetryPolicy = createMockTableRetryPolicy();
    when(writeRetryPolicy.toConfig(any(), any())).thenReturn(createConfigPair(key));
    Map<String, String> tableConfig = new RemoteTableDescriptor("1").withReadFunction(readFn).withWriteFunction(writeFn).withRateLimiter(rateLimiter, readCredFn, writeCredFn).withReadRetryPolicy(readRetryPolicy).withWriteRetryPolicy(writeRetryPolicy).toConfig(new MapConfig());
    verify(readFn, times(1)).toConfig(any(), any());
    verify(writeFn, times(1)).toConfig(any(), any());
    verify((TablePart) rateLimiter, times(1)).toConfig(any(), any());
    verify(readCredFn, times(1)).toConfig(any(), any());
    verify(writeCredFn, times(1)).toConfig(any(), any());
    verify(readRetryPolicy, times(1)).toConfig(any(), any());
    verify(writeRetryPolicy, times(1)).toConfig(any(), any());
    Assert.assertEquals(tableConfig.get("tables.1.io.read.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.ratelimiter.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.read.credit.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.credit.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.read.retry.policy.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.retry.policy.k0"), "v0");
}

Example 4 with TableRetryPolicy

use of org.apache.samza.table.retry.TableRetryPolicy in project samza by apache.

the class TestRemoteTable method getTable.

private <K, V, U, T extends RemoteTable<K, V, U>> T getTable(String tableId, TableReadFunction<K, V> readFn, TableWriteFunction<K, V, U> writeFn, ExecutorService cbExecutor, boolean retry) {
    TableRateLimiter<K, V> readRateLimiter = mock(TableRateLimiter.class);
    TableRateLimiter<K, V> writeRateLimiter = mock(TableRateLimiter.class);
    TableRateLimiter<K, U> updateRateLimiter = mock(TableRateLimiter.class);
    TableRetryPolicy readPolicy = retry ? new TableRetryPolicy() : null;
    TableRetryPolicy writePolicy = retry ? new TableRetryPolicy() : null;
    ExecutorService rateLimitingExecutor = Executors.newSingleThreadExecutor();
    ScheduledExecutorService retryExecutor = Executors.newSingleThreadScheduledExecutor();
    RemoteTable<K, V, U> table = new RemoteTable<>(tableId, readFn, writeFn, readRateLimiter, writeRateLimiter, updateRateLimiter, rateLimitingExecutor, readPolicy, writePolicy, retryExecutor, null, null, cbExecutor);
    table.init(getMockContext());
    if (readFn != null) {
        verify(readFn, times(1)).init(any(), any());
    }
    if (writeFn != null) {
        verify(writeFn, times(1)).init(any(), any());
    }
    return (T) table;
}

Example 5 with TableRetryPolicy

use of org.apache.samza.table.retry.TableRetryPolicy in project samza by apache.

the class RemoteTableProvider method getTable.

@Override
public ReadWriteUpdateTable getTable() {
    Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId));
    JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig());
    // Read part
    TableReadFunction readFn = deserializeObject(tableConfig, RemoteTableDescriptor.READ_FN);
    RateLimiter rateLimiter = deserializeObject(tableConfig, RemoteTableDescriptor.RATE_LIMITER);
    if (rateLimiter != null) {
        rateLimiter.init(this.context);
    }
    TableRateLimiter.CreditFunction<?, ?> readCreditFn = deserializeObject(tableConfig, RemoteTableDescriptor.READ_CREDIT_FN);
    TableRateLimiter readRateLimiter = rateLimiter != null && rateLimiter.getSupportedTags().contains(RemoteTableDescriptor.RL_READ_TAG) ? new TableRateLimiter(tableId, rateLimiter, readCreditFn, RemoteTableDescriptor.RL_READ_TAG) : null;
    TableRetryPolicy readRetryPolicy = deserializeObject(tableConfig, RemoteTableDescriptor.READ_RETRY_POLICY);
    // Write part
    // Reuse write rate limiter for update
    TableRateLimiter writeRateLimiter = null;
    TableRetryPolicy writeRetryPolicy = null;
    TableWriteFunction writeFn = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_FN);
    if (writeFn != null) {
        TableRateLimiter.CreditFunction<?, ?> writeCreditFn = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_CREDIT_FN);
        writeRateLimiter = rateLimiter != null && rateLimiter.getSupportedTags().contains(RemoteTableDescriptor.RL_WRITE_TAG) ? new TableRateLimiter(tableId, rateLimiter, writeCreditFn, RemoteTableDescriptor.RL_WRITE_TAG) : null;
        writeRetryPolicy = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_RETRY_POLICY);
    }
    if (readRetryPolicy != null || writeRetryPolicy != null) {
        retryExecutor = createRetryExecutor();
    }
    // Optional executor for future callback/completion. Shared by both read and write operations.
    int callbackPoolSize = Integer.parseInt(tableConfig.getForTable(tableId, RemoteTableDescriptor.ASYNC_CALLBACK_POOL_SIZE, "-1"));
    if (callbackPoolSize > 0) {
        callbackExecutors.computeIfAbsent(tableId, (arg) -> Executors.newFixedThreadPool(callbackPoolSize, (runnable) -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-async-callback-pool");
            thread.setDaemon(true);
            return thread;
        }));
    }
    boolean isRateLimited = readRateLimiter != null || writeRateLimiter != null;
    if (isRateLimited) {
        rateLimitingExecutors.computeIfAbsent(tableId, (arg) -> Executors.newSingleThreadExecutor(runnable -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-async-executor");
            thread.setDaemon(true);
            return thread;
        }));
    }
    BatchProvider batchProvider = deserializeObject(tableConfig, RemoteTableDescriptor.BATCH_PROVIDER);
    if (batchProvider != null) {
        batchExecutors.computeIfAbsent(tableId, (arg) -> Executors.newSingleThreadScheduledExecutor(runnable -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-batch-scheduled-executor");
            thread.setDaemon(true);
            return thread;
        }));
    }
    RemoteTable table = new RemoteTable(tableId, readFn, writeFn, readRateLimiter, writeRateLimiter, writeRateLimiter, rateLimitingExecutors.get(tableId), readRetryPolicy, writeRetryPolicy, retryExecutor, batchProvider, batchExecutors.get(tableId), callbackExecutors.get(tableId));
    table.init(this.context);
    tables.add(table);
    return table;
}